packages feed

hpython (empty) → 0.1

raw patch · 96 files changed

+86036/−0 lines, 96 filesdep +basedep +bifunctorsdep +bytestringsetup-changed

Dependencies added: base, bifunctors, bytestring, containers, criterion, deepseq, deriving-compat, digit, dlist, filepath, fingertree, hedgehog, hpython, lens, megaparsec, mtl, parsers, parsers-megaparsec, semigroupoids, text, these, validation

Files

+ ChangeLog.md view
@@ -0,0 +1,7 @@+# Revision history for hpython++## 0.1++*2018-01-07*++Initial release
+ LICENCE view
@@ -0,0 +1,31 @@+Copyright (c) 2017, Commonwealth Scientific and Industrial Research Organisation+(CSIRO) ABN 41 687 119 230.++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Data61 nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ benchmarks/Main.hs view
@@ -0,0 +1,48 @@+{-# language DataKinds #-}+{-# options_ghc -ddump-to-file -ddump-simpl  #-}+module Main where++import Criterion.Main++import Data.List.NonEmpty (NonEmpty)+import Data.Validation (Validation(..))+import System.Exit (exitFailure)++import qualified Data.Text.IO as StrictText++import Language.Python.Parse (parseModule)+import Language.Python.Parse.Error (ParseError)+import Language.Python.Internal.Lexer (SrcInfo, tokenize)+import Language.Python.Validate++parseCheckSeq :: FilePath -> IO ()+parseCheckSeq name = do+  file <- StrictText.readFile name+  py <-+    case parseModule name file of+      Failure e -> print (e :: NonEmpty (ParseError SrcInfo)) *> exitFailure+      Success a -> pure a+  case runValidateIndentation $ validateModuleIndentation py of+    Failure errs ->+      print (errs :: NonEmpty (IndentationError SrcInfo)) *> exitFailure+    Success res ->+      case runValidateSyntax (validateModuleSyntax res) of+        Failure errs' ->+          print (errs' :: (NonEmpty (SyntaxError SrcInfo))) *> exitFailure+        Success a -> pure $ seq a ()++tokenizeSeq :: FilePath -> IO ()+tokenizeSeq name = do+  file <- StrictText.readFile name+  case tokenize name file of+    Left e -> print (e :: (ParseError SrcInfo)) *> exitFailure+    Right a -> pure $ seq (length a) ()++main :: IO ()+main =+  defaultMain+  [ bench "tokenize 9000 lines of correct python" $+    nfIO (tokenizeSeq "./benchmarks/pypy.py")+  , bench "9000 lines of correct python" $+    nfIO (parseCheckSeq "./benchmarks/pypy.py")+  ]
+ benchmarks/pypy.py view
@@ -0,0 +1,6399 @@+# Copyright (c) 2004 Python Software Foundation.+# All rights reserved.++# Written by Eric Price <eprice at tjhsst.edu>+#    and Facundo Batista <facundo at taniquetil.com.ar>+#    and Raymond Hettinger <python at rcn.com>+#    and Aahz <aahz at pobox.com>+#    and Tim Peters++# This module should be kept in sync with the latest updates of the+# IBM specification as it evolves.  Those updates will be treated+# as bug fixes (deviation from the spec is a compatibility, usability+# bug) and will be backported.  At this point the spec is stabilizing+# and the updates are becoming fewer, smaller, and less significant.++"""+This is an implementation of decimal floating point arithmetic based on+the General Decimal Arithmetic Specification:++    http://speleotrove.com/decimal/decarith.html++and IEEE standard 854-1987:++    http://en.wikipedia.org/wiki/IEEE_854-1987++Decimal floating point has finite precision with arbitrarily large bounds.++The purpose of this module is to support arithmetic using familiar+"schoolhouse" rules and to avoid some of the tricky representation+issues associated with binary floating point.  The package is especially+useful for financial applications or for contexts where users have+expectations that are at odds with binary floating point (for instance,+in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead+of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected+Decimal('0.00')).++Here are some examples of using the decimal module:++>>> from decimal import *+>>> setcontext(ExtendedContext)+>>> Decimal(0)+Decimal('0')+>>> Decimal('1')+Decimal('1')+>>> Decimal('-.0123')+Decimal('-0.0123')+>>> Decimal(123456)+Decimal('123456')+>>> Decimal('123.45e12345678')+Decimal('1.2345E+12345680')+>>> Decimal('1.33') + Decimal('1.27')+Decimal('2.60')+>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')+Decimal('-2.20')+>>> dig = Decimal(1)+>>> print(dig / Decimal(3))+0.333333333+>>> getcontext().prec = 18+>>> print(dig / Decimal(3))+0.333333333333333333+>>> print(dig.sqrt())+1+>>> print(Decimal(3).sqrt())+1.73205080756887729+>>> print(Decimal(3) ** 123)+4.85192780976896427E+58+>>> inf = Decimal(1) / Decimal(0)+>>> print(inf)+Infinity+>>> neginf = Decimal(-1) / Decimal(0)+>>> print(neginf)+-Infinity+>>> print(neginf + inf)+NaN+>>> print(neginf * inf)+-Infinity+>>> print(dig / 0)+Infinity+>>> getcontext().traps[DivisionByZero] = 1+>>> print(dig / 0)+Traceback (most recent call last):+  ...+  ...+  ...+decimal.DivisionByZero: x / 0+>>> c = Context()+>>> c.traps[InvalidOperation] = 0+>>> print(c.flags[InvalidOperation])+0+>>> c.divide(Decimal(0), Decimal(0))+Decimal('NaN')+>>> c.traps[InvalidOperation] = 1+>>> print(c.flags[InvalidOperation])+1+>>> c.flags[InvalidOperation] = 0+>>> print(c.flags[InvalidOperation])+0+>>> print(c.divide(Decimal(0), Decimal(0)))+Traceback (most recent call last):+  ...+  ...+  ...+decimal.InvalidOperation: 0 / 0+>>> print(c.flags[InvalidOperation])+1+>>> c.flags[InvalidOperation] = 0+>>> c.traps[InvalidOperation] = 0+>>> print(c.divide(Decimal(0), Decimal(0)))+NaN+>>> print(c.flags[InvalidOperation])+1+>>>+"""++__all__ = [+    # Two major classes+    'Decimal', 'Context',++    # Named tuple representation+    'DecimalTuple',++    # Contexts+    'DefaultContext', 'BasicContext', 'ExtendedContext',++    # Exceptions+    'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero',+    'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow',+    'FloatOperation',++    # Exceptional conditions that trigger InvalidOperation+    'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined',++    # Constants for use in setting up contexts+    'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING',+    'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP',++    # Functions for manipulating contexts+    'setcontext', 'getcontext', 'localcontext',++    # Limits for the C version for compatibility+    'MAX_PREC',  'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',++    # C version: compile time choice that enables the thread local context+    'HAVE_THREADS'+]++__xname__ = __name__    # sys.modules lookup (--without-threads)+__name__ = 'decimal'    # For pickling+__version__ = '1.70'    # Highest version of the spec this complies with+                        # See http://speleotrove.com/decimal/+__libmpdec_version__ = "2.4.1" # compatible libmpdec version++import math as _math+import numbers as _numbers+import sys++try:+    from collections import namedtuple as _namedtuple+    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')+except ImportError:+    DecimalTuple = lambda *args: args++# Rounding+ROUND_DOWN = 'ROUND_DOWN'+ROUND_HALF_UP = 'ROUND_HALF_UP'+ROUND_HALF_EVEN = 'ROUND_HALF_EVEN'+ROUND_CEILING = 'ROUND_CEILING'+ROUND_FLOOR = 'ROUND_FLOOR'+ROUND_UP = 'ROUND_UP'+ROUND_HALF_DOWN = 'ROUND_HALF_DOWN'+ROUND_05UP = 'ROUND_05UP'++# Compatibility with the C version+HAVE_THREADS = True+if sys.maxsize == 2**63-1:+    MAX_PREC = 999999999999999999+    MAX_EMAX = 999999999999999999+    MIN_EMIN = -999999999999999999+else:+    MAX_PREC = 425000000+    MAX_EMAX = 425000000+    MIN_EMIN = -425000000++MIN_ETINY = MIN_EMIN - (MAX_PREC-1)++# Errors++class DecimalException(ArithmeticError):+    """Base exception class.++    Used exceptions derive from this.+    If an exception derives from another exception besides this (such as+    Underflow (Inexact, Rounded, Subnormal) that indicates that it is only+    called if the others are present.  This isn't actually used for+    anything, though.++    handle  -- Called when context._raise_error is called and the+               trap_enabler is not set.  First argument is self, second is the+               context.  More arguments can be given, those being after+               the explanation in _raise_error (For example,+               context._raise_error(NewError, '(-x)!', self._sign) would+               call NewError().handle(context, self._sign).)++    To define a new exception, it should be sufficient to have it derive+    from DecimalException.+    """+    def handle(self, context, *args):+        pass+++class Clamped(DecimalException):+    """Exponent of a 0 changed to fit bounds.++    This occurs and signals clamped if the exponent of a result has been+    altered in order to fit the constraints of a specific concrete+    representation.  This may occur when the exponent of a zero result would+    be outside the bounds of a representation, or when a large normal+    number would have an encoded exponent that cannot be represented.  In+    this latter case, the exponent is reduced to fit and the corresponding+    number of zero digits are appended to the coefficient ("fold-down").+    """++class InvalidOperation(DecimalException):+    """An invalid operation was performed.++    Various bad things cause this:++    Something creates a signaling NaN+    -INF + INF+    0 * (+-)INF+    (+-)INF / (+-)INF+    x % 0+    (+-)INF % x+    x._rescale( non-integer )+    sqrt(-x) , x > 0+    0 ** 0+    x ** (non-integer)+    x ** (+-)INF+    An operand is invalid++    The result of the operation after these is a quiet positive NaN,+    except when the cause is a signaling NaN, in which case the result is+    also a quiet NaN, but with the original sign, and an optional+    diagnostic information.+    """+    def handle(self, context, *args):+        if args:+            ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True)+            return ans._fix_nan(context)+        return _NaN++class ConversionSyntax(InvalidOperation):+    """Trying to convert badly formed string.++    This occurs and signals invalid-operation if a string is being+    converted to a number and it does not conform to the numeric string+    syntax.  The result is [0,qNaN].+    """+    def handle(self, context, *args):+        return _NaN++class DivisionByZero(DecimalException, ZeroDivisionError):+    """Division by 0.++    This occurs and signals division-by-zero if division of a finite number+    by zero was attempted (during a divide-integer or divide operation, or a+    power operation with negative right-hand operand), and the dividend was+    not zero.++    The result of the operation is [sign,inf], where sign is the exclusive+    or of the signs of the operands for divide, or is 1 for an odd power of+    -0, for power.+    """++    def handle(self, context, sign, *args):+        return _SignedInfinity[sign]++class DivisionImpossible(InvalidOperation):+    """Cannot perform the division adequately.++    This occurs and signals invalid-operation if the integer result of a+    divide-integer or remainder operation had too many digits (would be+    longer than precision).  The result is [0,qNaN].+    """++    def handle(self, context, *args):+        return _NaN++class DivisionUndefined(InvalidOperation, ZeroDivisionError):+    """Undefined result of division.++    This occurs and signals invalid-operation if division by zero was+    attempted (during a divide-integer, divide, or remainder operation), and+    the dividend is also zero.  The result is [0,qNaN].+    """++    def handle(self, context, *args):+        return _NaN++class Inexact(DecimalException):+    """Had to round, losing information.++    This occurs and signals inexact whenever the result of an operation is+    not exact (that is, it needed to be rounded and any discarded digits+    were non-zero), or if an overflow or underflow condition occurs.  The+    result in all cases is unchanged.++    The inexact signal may be tested (or trapped) to determine if a given+    operation (or sequence of operations) was inexact.+    """++class InvalidContext(InvalidOperation):+    """Invalid context.  Unknown rounding, for example.++    This occurs and signals invalid-operation if an invalid context was+    detected during an operation.  This can occur if contexts are not checked+    on creation and either the precision exceeds the capability of the+    underlying concrete representation or an unknown or unsupported rounding+    was specified.  These aspects of the context need only be checked when+    the values are required to be used.  The result is [0,qNaN].+    """++    def handle(self, context, *args):+        return _NaN++class Rounded(DecimalException):+    """Number got rounded (not  necessarily changed during rounding).++    This occurs and signals rounded whenever the result of an operation is+    rounded (that is, some zero or non-zero digits were discarded from the+    coefficient), or if an overflow or underflow condition occurs.  The+    result in all cases is unchanged.++    The rounded signal may be tested (or trapped) to determine if a given+    operation (or sequence of operations) caused a loss of precision.+    """++class Subnormal(DecimalException):+    """Exponent < Emin before rounding.++    This occurs and signals subnormal whenever the result of a conversion or+    operation is subnormal (that is, its adjusted exponent is less than+    Emin, before any rounding).  The result in all cases is unchanged.++    The subnormal signal may be tested (or trapped) to determine if a given+    or operation (or sequence of operations) yielded a subnormal result.+    """++class Overflow(Inexact, Rounded):+    """Numerical overflow.++    This occurs and signals overflow if the adjusted exponent of a result+    (from a conversion or from an operation that is not an attempt to divide+    by zero), after rounding, would be greater than the largest value that+    can be handled by the implementation (the value Emax).++    The result depends on the rounding mode:++    For round-half-up and round-half-even (and for round-half-down and+    round-up, if implemented), the result of the operation is [sign,inf],+    where sign is the sign of the intermediate result.  For round-down, the+    result is the largest finite number that can be represented in the+    current precision, with the sign of the intermediate result.  For+    round-ceiling, the result is the same as for round-down if the sign of+    the intermediate result is 1, or is [0,inf] otherwise.  For round-floor,+    the result is the same as for round-down if the sign of the intermediate+    result is 0, or is [1,inf] otherwise.  In all cases, Inexact and Rounded+    will also be raised.+    """++    def handle(self, context, sign, *args):+        if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN,+                                ROUND_HALF_DOWN, ROUND_UP):+            return _SignedInfinity[sign]+        if sign == 0:+            if context.rounding == ROUND_CEILING:+                return _SignedInfinity[sign]+            return _dec_from_triple(sign, '9'*context.prec,+                            context.Emax-context.prec+1)+        if sign == 1:+            if context.rounding == ROUND_FLOOR:+                return _SignedInfinity[sign]+            return _dec_from_triple(sign, '9'*context.prec,+                             context.Emax-context.prec+1)+++class Underflow(Inexact, Rounded, Subnormal):+    """Numerical underflow with result rounded to 0.++    This occurs and signals underflow if a result is inexact and the+    adjusted exponent of the result would be smaller (more negative) than+    the smallest value that can be handled by the implementation (the value+    Emin).  That is, the result is both inexact and subnormal.++    The result after an underflow will be a subnormal number rounded, if+    necessary, so that its exponent is not less than Etiny.  This may result+    in 0 with the sign of the intermediate result and an exponent of Etiny.++    In all cases, Inexact, Rounded, and Subnormal will also be raised.+    """++class FloatOperation(DecimalException, TypeError):+    """Enable stricter semantics for mixing floats and Decimals.++    If the signal is not trapped (default), mixing floats and Decimals is+    permitted in the Decimal() constructor, context.create_decimal() and+    all comparison operators. Both conversion and comparisons are exact.+    Any occurrence of a mixed operation is silently recorded by setting+    FloatOperation in the context flags.  Explicit conversions with+    Decimal.from_float() or context.create_decimal_from_float() do not+    set the flag.++    Otherwise (the signal is trapped), only equality comparisons and explicit+    conversions are silent. All other mixed operations raise FloatOperation.+    """++# List of public traps and flags+_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded,+            Underflow, InvalidOperation, Subnormal, FloatOperation]++# Map conditions (per the spec) to signals+_condition_map = {ConversionSyntax:InvalidOperation,+                  DivisionImpossible:InvalidOperation,+                  DivisionUndefined:InvalidOperation,+                  InvalidContext:InvalidOperation}++# Valid rounding modes+_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,+                   ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP)++##### Context Functions ##################################################++# The getcontext() and setcontext() function manage access to a thread-local+# current context.  Py2.4 offers direct support for thread locals.  If that+# is not available, use threading.current_thread() which is slower but will+# work for older Pythons.  If threads are not part of the build, create a+# mock threading object with threading.local() returning the module namespace.++try:+    import threading+except ImportError:+    # Python was compiled without threads; create a mock object instead+    class MockThreading(object):+        def local(self, sys=sys):+            return sys.modules[__xname__]+    threading = MockThreading()+    del MockThreading++try:+    threading.local++except AttributeError:++    # To fix reloading, force it to create a new context+    # Old contexts have different exceptions in their dicts, making problems.+    if hasattr(threading.current_thread(), '__decimal_context__'):+        del threading.current_thread().__decimal_context__++    def setcontext(context):+        """Set this thread's context to context."""+        if context in (DefaultContext, BasicContext, ExtendedContext):+            context = context.copy()+            context.clear_flags()+        threading.current_thread().__decimal_context__ = context++    def getcontext():+        """Returns this thread's context.++        If this thread does not yet have a context, returns+        a new context and sets this thread's context.+        New contexts are copies of DefaultContext.+        """+        try:+            return threading.current_thread().__decimal_context__+        except AttributeError:+            context = Context()+            threading.current_thread().__decimal_context__ = context+            return context++else:++    local = threading.local()+    if hasattr(local, '__decimal_context__'):+        del local.__decimal_context__++    def getcontext(_local=local):+        """Returns this thread's context.++        If this thread does not yet have a context, returns+        a new context and sets this thread's context.+        New contexts are copies of DefaultContext.+        """+        try:+            return _local.__decimal_context__+        except AttributeError:+            context = Context()+            _local.__decimal_context__ = context+            return context++    def setcontext(context, _local=local):+        """Set this thread's context to context."""+        if context in (DefaultContext, BasicContext, ExtendedContext):+            context = context.copy()+            context.clear_flags()+        _local.__decimal_context__ = context++    del threading, local        # Don't contaminate the namespace++def localcontext(ctx=None):+    """Return a context manager for a copy of the supplied context++    Uses a copy of the current context if no context is specified+    The returned context manager creates a local decimal context+    in a with statement:+        def sin(x):+             with localcontext() as ctx:+                 ctx.prec += 2+                 # Rest of sin calculation algorithm+                 # uses a precision 2 greater than normal+             return +s  # Convert result to normal precision++         def sin(x):+             with localcontext(ExtendedContext):+                 # Rest of sin calculation algorithm+                 # uses the Extended Context from the+                 # General Decimal Arithmetic Specification+             return +s  # Convert result to normal context++    >>> setcontext(DefaultContext)+    >>> print(getcontext().prec)+    28+    >>> with localcontext():+    ...     ctx = getcontext()+    ...     ctx.prec += 2+    ...     print(ctx.prec)+    ...+    30+    >>> with localcontext(ExtendedContext):+    ...     print(getcontext().prec)+    ...+    9+    >>> print(getcontext().prec)+    28+    """+    if ctx is None: ctx = getcontext()+    return _ContextManager(ctx)+++##### Decimal class #######################################################++# Do not subclass Decimal from numbers.Real and do not register it as such+# (because Decimals are not interoperable with floats).  See the notes in+# numbers.py for more detail.++class Decimal(object):+    """Floating point class for decimal arithmetic."""++    __slots__ = ('_exp','_int','_sign', '_is_special')+    # Generally, the value of the Decimal instance is given by+    #  (-1)**_sign * _int * 10**_exp+    # Special values are signified by _is_special == True++    # We're immutable, so use __new__ not __init__+    def __new__(cls, value="0", context=None):+        """Create a decimal point instance.++        >>> Decimal('3.14')              # string input+        Decimal('3.14')+        >>> Decimal((0, (3, 1, 4), -2))  # tuple (sign, digit_tuple, exponent)+        Decimal('3.14')+        >>> Decimal(314)                 # int+        Decimal('314')+        >>> Decimal(Decimal(314))        # another decimal instance+        Decimal('314')+        >>> Decimal('  3.14  \\n')        # leading and trailing whitespace okay+        Decimal('3.14')+        """++        # Note that the coefficient, self._int, is actually stored as+        # a string rather than as a tuple of digits.  This speeds up+        # the "digits to integer" and "integer to digits" conversions+        # that are used in almost every arithmetic operation on+        # Decimals.  This is an internal detail: the as_tuple function+        # and the Decimal constructor still deal with tuples of+        # digits.++        self = object.__new__(cls)++        # From a string+        # REs insist on real strings, so we can too.+        if isinstance(value, str):+            m = _parser(value.strip())+            if m is None:+                if context is None:+                    context = getcontext()+                return context._raise_error(ConversionSyntax,+                                "Invalid literal for Decimal: %r" % value)++            if m.group('sign') == "-":+                self._sign = 1+            else:+                self._sign = 0+            intpart = m.group('int')+            if intpart is not None:+                # finite number+                fracpart = m.group('frac') or ''+                exp = int(m.group('exp') or '0')+                self._int = str(int(intpart+fracpart))+                self._exp = exp - len(fracpart)+                self._is_special = False+            else:+                diag = m.group('diag')+                if diag is not None:+                    # NaN+                    self._int = str(int(diag or '0')).lstrip('0')+                    if m.group('signal'):+                        self._exp = 'N'+                    else:+                        self._exp = 'n'+                else:+                    # infinity+                    self._int = '0'+                    self._exp = 'F'+                self._is_special = True+            return self++        # From an integer+        if isinstance(value, int):+            if value >= 0:+                self._sign = 0+            else:+                self._sign = 1+            self._exp = 0+            self._int = str(abs(value))+            self._is_special = False+            return self++        # From another decimal+        if isinstance(value, Decimal):+            self._exp  = value._exp+            self._sign = value._sign+            self._int  = value._int+            self._is_special  = value._is_special+            return self++        # From an internal working value+        if isinstance(value, _WorkRep):+            self._sign = value.sign+            self._int = str(value.int)+            self._exp = int(value.exp)+            self._is_special = False+            return self++        # tuple/list conversion (possibly from as_tuple())+        if isinstance(value, (list,tuple)):+            if len(value) != 3:+                raise ValueError('Invalid tuple size in creation of Decimal '+                                 'from list or tuple.  The list or tuple '+                                 'should have exactly three elements.')+            # process sign.  The isinstance test rejects floats+            if not (isinstance(value[0], int) and value[0] in (0,1)):+                raise ValueError("Invalid sign.  The first value in the tuple "+                                 "should be an integer; either 0 for a "+                                 "positive number or 1 for a negative number.")+            self._sign = value[0]+            if value[2] == 'F':+                # infinity: value[1] is ignored+                self._int = '0'+                self._exp = value[2]+                self._is_special = True+            else:+                # process and validate the digits in value[1]+                digits = []+                for digit in value[1]:+                    if isinstance(digit, int) and 0 <= digit <= 9:+                        # skip leading zeros+                        if digits or digit != 0:+                            digits.append(digit)+                    else:+                        raise ValueError("The second value in the tuple must "+                                         "be composed of integers in the range "+                                         "0 through 9.")+                if value[2] in ('n', 'N'):+                    # NaN: digits form the diagnostic+                    self._int = ''.join(map(str, digits))+                    self._exp = value[2]+                    self._is_special = True+                elif isinstance(value[2], int):+                    # finite number: digits give the coefficient+                    self._int = ''.join(map(str, digits or [0]))+                    self._exp = value[2]+                    self._is_special = False+                else:+                    raise ValueError("The third value in the tuple must "+                                     "be an integer, or one of the "+                                     "strings 'F', 'n', 'N'.")+            return self++        if isinstance(value, float):+            if context is None:+                context = getcontext()+            context._raise_error(FloatOperation,+                "strict semantics for mixing floats and Decimals are "+                "enabled")+            value = Decimal.from_float(value)+            self._exp  = value._exp+            self._sign = value._sign+            self._int  = value._int+            self._is_special  = value._is_special+            return self++        raise TypeError("Cannot convert %r to Decimal" % value)++    @classmethod+    def from_float(cls, f):+        """Converts a float to a decimal number, exactly.++        Note that Decimal.from_float(0.1) is not the same as Decimal('0.1').+        Since 0.1 is not exactly representable in binary floating point, the+        value is stored as the nearest representable value which is+        0x1.999999999999ap-4.  The exact equivalent of the value in decimal+        is 0.1000000000000000055511151231257827021181583404541015625.++        >>> Decimal.from_float(0.1)+        Decimal('0.1000000000000000055511151231257827021181583404541015625')+        >>> Decimal.from_float(float('nan'))+        Decimal('NaN')+        >>> Decimal.from_float(float('inf'))+        Decimal('Infinity')+        >>> Decimal.from_float(-float('inf'))+        Decimal('-Infinity')+        >>> Decimal.from_float(-0.0)+        Decimal('-0')++        """+        if isinstance(f, int):                # handle integer inputs+            return cls(f)+        if not isinstance(f, float):+            raise TypeError("argument must be int or float.")+        if _math.isinf(f) or _math.isnan(f):+            return cls(repr(f))+        if _math.copysign(1.0, f) == 1.0:+            sign = 0+        else:+            sign = 1+        n, d = abs(f).as_integer_ratio()+        k = d.bit_length() - 1+        result = _dec_from_triple(sign, str(n*5**k), -k)+        if cls is Decimal:+            return result+        else:+            return cls(result)++    def _isnan(self):+        """Returns whether the number is not actually one.++        0 if a number+        1 if NaN+        2 if sNaN+        """+        if self._is_special:+            exp = self._exp+            if exp == 'n':+                return 1+            elif exp == 'N':+                return 2+        return 0++    def _isinfinity(self):+        """Returns whether the number is infinite++        0 if finite or not a number+        1 if +INF+        -1 if -INF+        """+        if self._exp == 'F':+            if self._sign:+                return -1+            return 1+        return 0++    def _check_nans(self, other=None, context=None):+        """Returns whether the number is not actually one.++        if self, other are sNaN, signal+        if self, other are NaN return nan+        return 0++        Done before operations.+        """++        self_is_nan = self._isnan()+        if other is None:+            other_is_nan = False+        else:+            other_is_nan = other._isnan()++        if self_is_nan or other_is_nan:+            if context is None:+                context = getcontext()++            if self_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        self)+            if other_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        other)+            if self_is_nan:+                return self._fix_nan(context)++            return other._fix_nan(context)+        return 0++    def _compare_check_nans(self, other, context):+        """Version of _check_nans used for the signaling comparisons+        compare_signal, __le__, __lt__, __ge__, __gt__.++        Signal InvalidOperation if either self or other is a (quiet+        or signaling) NaN.  Signaling NaNs take precedence over quiet+        NaNs.++        Return 0 if neither operand is a NaN.++        """+        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            if self.is_snan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving sNaN',+                                            self)+            elif other.is_snan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving sNaN',+                                            other)+            elif self.is_qnan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving NaN',+                                            self)+            elif other.is_qnan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving NaN',+                                            other)+        return 0++    def __bool__(self):+        """Return True if self is nonzero; otherwise return False.++        NaNs and infinities are considered nonzero.+        """+        return self._is_special or self._int != '0'++    def _cmp(self, other):+        """Compare the two non-NaN decimal instances self and other.++        Returns -1 if self < other, 0 if self == other and 1+        if self > other.  This routine is for internal use only."""++        if self._is_special or other._is_special:+            self_inf = self._isinfinity()+            other_inf = other._isinfinity()+            if self_inf == other_inf:+                return 0+            elif self_inf < other_inf:+                return -1+            else:+                return 1++        # check for zeros;  Decimal('0') == Decimal('-0')+        if not self:+            if not other:+                return 0+            else:+                return -((-1)**other._sign)+        if not other:+            return (-1)**self._sign++        # If different signs, neg one is less+        if other._sign < self._sign:+            return -1+        if self._sign < other._sign:+            return 1++        self_adjusted = self.adjusted()+        other_adjusted = other.adjusted()+        if self_adjusted == other_adjusted:+            self_padded = self._int + '0'*(self._exp - other._exp)+            other_padded = other._int + '0'*(other._exp - self._exp)+            if self_padded == other_padded:+                return 0+            elif self_padded < other_padded:+                return -(-1)**self._sign+            else:+                return (-1)**self._sign+        elif self_adjusted > other_adjusted:+            return (-1)**self._sign+        else: # self_adjusted < other_adjusted+            return -((-1)**self._sign)++    # Note: The Decimal standard doesn't cover rich comparisons for+    # Decimals.  In particular, the specification is silent on the+    # subject of what should happen for a comparison involving a NaN.+    # We take the following approach:+    #+    #   == comparisons involving a quiet NaN always return False+    #   != comparisons involving a quiet NaN always return True+    #   == or != comparisons involving a signaling NaN signal+    #      InvalidOperation, and return False or True as above if the+    #      InvalidOperation is not trapped.+    #   <, >, <= and >= comparisons involving a (quiet or signaling)+    #      NaN signal InvalidOperation, and return False if the+    #      InvalidOperation is not trapped.+    #+    # This behavior is designed to conform as closely as possible to+    # that specified by IEEE 754.++    def __eq__(self, other, context=None):+        self, other = _convert_for_comparison(self, other, equality_op=True)+        if other is NotImplemented:+            return other+        if self._check_nans(other, context):+            return False+        return self._cmp(other) == 0++    def __lt__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) < 0++    def __le__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) <= 0++    def __gt__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) > 0++    def __ge__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) >= 0++    def compare(self, other, context=None):+        """Compare self to other.  Return a decimal value:++        a or b is a NaN ==> Decimal('NaN')+        a < b           ==> Decimal('-1')+        a == b          ==> Decimal('0')+        a > b           ==> Decimal('1')+        """+        other = _convert_other(other, raiseit=True)++        # Compare(NaN, NaN) = NaN+        if (self._is_special or other and other._is_special):+            ans = self._check_nans(other, context)+            if ans:+                return ans++        return Decimal(self._cmp(other))++    def __hash__(self):+        """x.__hash__() <==> hash(x)"""++        # In order to make sure that the hash of a Decimal instance+        # agrees with the hash of a numerically equal integer, float+        # or Fraction, we follow the rules for numeric hashes outlined+        # in the documentation.  (See library docs, 'Built-in Types').+        if self._is_special:+            if self.is_snan():+                raise TypeError('Cannot hash a signaling NaN value.')+            elif self.is_nan():+                return _PyHASH_NAN+            else:+                if self._sign:+                    return -_PyHASH_INF+                else:+                    return _PyHASH_INF++        if self._exp >= 0:+            exp_hash = pow(10, self._exp, _PyHASH_MODULUS)+        else:+            exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS)+        hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS+        ans = hash_ if self >= 0 else -hash_+        return -2 if ans == -1 else ans++    def as_tuple(self):+        """Represents the number as a triple tuple.++        To show the internals exactly as they are.+        """+        return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp)++    def __repr__(self):+        """Represents the number as an instance of Decimal."""+        # Invariant:  eval(repr(d)) == d+        return "Decimal('%s')" % str(self)++    def __str__(self, eng=False, context=None):+        """Return string representation of the number in scientific notation.++        Captures all of the information in the underlying representation.+        """++        sign = ['', '-'][self._sign]+        if self._is_special:+            if self._exp == 'F':+                return sign + 'Infinity'+            elif self._exp == 'n':+                return sign + 'NaN' + self._int+            else: # self._exp == 'N'+                return sign + 'sNaN' + self._int++        # number of digits of self._int to left of decimal point+        leftdigits = self._exp + len(self._int)++        # dotplace is number of digits of self._int to the left of the+        # decimal point in the mantissa of the output string (that is,+        # after adjusting the exponent)+        if self._exp <= 0 and leftdigits > -6:+            # no exponent required+            dotplace = leftdigits+        elif not eng:+            # usual scientific notation: 1 digit on left of the point+            dotplace = 1+        elif self._int == '0':+            # engineering notation, zero+            dotplace = (leftdigits + 1) % 3 - 1+        else:+            # engineering notation, nonzero+            dotplace = (leftdigits - 1) % 3 + 1++        if dotplace <= 0:+            intpart = '0'+            fracpart = '.' + '0'*(-dotplace) + self._int+        elif dotplace >= len(self._int):+            intpart = self._int+'0'*(dotplace-len(self._int))+            fracpart = ''+        else:+            intpart = self._int[:dotplace]+            fracpart = '.' + self._int[dotplace:]+        if leftdigits == dotplace:+            exp = ''+        else:+            if context is None:+                context = getcontext()+            exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace)++        return sign + intpart + fracpart + exp++    def to_eng_string(self, context=None):+        """Convert to a string, using engineering notation if an exponent is needed.++        Engineering notation has an exponent which is a multiple of 3.  This+        can leave up to 3 digits to the left of the decimal place and may+        require the addition of either one or two trailing zeros.+        """+        return self.__str__(eng=True, context=context)++    def __neg__(self, context=None):+        """Returns a copy with the sign switched.++        Rounds, if it has reason.+        """+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        if context is None:+            context = getcontext()++        if not self and context.rounding != ROUND_FLOOR:+            # -Decimal('0') is Decimal('0'), not Decimal('-0'), except+            # in ROUND_FLOOR rounding mode.+            ans = self.copy_abs()+        else:+            ans = self.copy_negate()++        return ans._fix(context)++    def __pos__(self, context=None):+        """Returns a copy, unless it is a sNaN.++        Rounds the number (if more than precision digits)+        """+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        if context is None:+            context = getcontext()++        if not self and context.rounding != ROUND_FLOOR:+            # + (-0) = 0, except in ROUND_FLOOR rounding mode.+            ans = self.copy_abs()+        else:+            ans = Decimal(self)++        return ans._fix(context)++    def __abs__(self, round=True, context=None):+        """Returns the absolute value of self.++        If the keyword argument 'round' is false, do not round.  The+        expression self.__abs__(round=False) is equivalent to+        self.copy_abs().+        """+        if not round:+            return self.copy_abs()++        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        if self._sign:+            ans = self.__neg__(context=context)+        else:+            ans = self.__pos__(context=context)++        return ans++    def __add__(self, other, context=None):+        """Returns self + other.++        -INF + INF (or the reverse) cause InvalidOperation errors.+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context)+            if ans:+                return ans++            if self._isinfinity():+                # If both INF, same sign => same as both, opposite => error.+                if self._sign != other._sign and other._isinfinity():+                    return context._raise_error(InvalidOperation, '-INF + INF')+                return Decimal(self)+            if other._isinfinity():+                return Decimal(other)  # Can't both be infinity here++        exp = min(self._exp, other._exp)+        negativezero = 0+        if context.rounding == ROUND_FLOOR and self._sign != other._sign:+            # If the answer is 0, the sign should be negative, in this case.+            negativezero = 1++        if not self and not other:+            sign = min(self._sign, other._sign)+            if negativezero:+                sign = 1+            ans = _dec_from_triple(sign, '0', exp)+            ans = ans._fix(context)+            return ans+        if not self:+            exp = max(exp, other._exp - context.prec-1)+            ans = other._rescale(exp, context.rounding)+            ans = ans._fix(context)+            return ans+        if not other:+            exp = max(exp, self._exp - context.prec-1)+            ans = self._rescale(exp, context.rounding)+            ans = ans._fix(context)+            return ans++        op1 = _WorkRep(self)+        op2 = _WorkRep(other)+        op1, op2 = _normalize(op1, op2, context.prec)++        result = _WorkRep()+        if op1.sign != op2.sign:+            # Equal and opposite+            if op1.int == op2.int:+                ans = _dec_from_triple(negativezero, '0', exp)+                ans = ans._fix(context)+                return ans+            if op1.int < op2.int:+                op1, op2 = op2, op1+                # OK, now abs(op1) > abs(op2)+            if op1.sign == 1:+                result.sign = 1+                op1.sign, op2.sign = op2.sign, op1.sign+            else:+                result.sign = 0+                # So we know the sign, and op1 > 0.+        elif op1.sign == 1:+            result.sign = 1+            op1.sign, op2.sign = (0, 0)+        else:+            result.sign = 0+        # Now, op1 > abs(op2) > 0++        if op2.sign == 0:+            result.int = op1.int + op2.int+        else:+            result.int = op1.int - op2.int++        result.exp = op1.exp+        ans = Decimal(result)+        ans = ans._fix(context)+        return ans++    __radd__ = __add__++    def __sub__(self, other, context=None):+        """Return self - other"""+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context=context)+            if ans:+                return ans++        # self - other is computed as self + other.copy_negate()+        return self.__add__(other.copy_negate(), context=context)++    def __rsub__(self, other, context=None):+        """Return other - self"""+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        return other.__sub__(self, context=context)++    def __mul__(self, other, context=None):+        """Return self * other.++        (+-) INF * 0 (or its reverse) raise InvalidOperation.+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        resultsign = self._sign ^ other._sign++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context)+            if ans:+                return ans++            if self._isinfinity():+                if not other:+                    return context._raise_error(InvalidOperation, '(+-)INF * 0')+                return _SignedInfinity[resultsign]++            if other._isinfinity():+                if not self:+                    return context._raise_error(InvalidOperation, '0 * (+-)INF')+                return _SignedInfinity[resultsign]++        resultexp = self._exp + other._exp++        # Special case for multiplying by zero+        if not self or not other:+            ans = _dec_from_triple(resultsign, '0', resultexp)+            # Fixing in case the exponent is out of bounds+            ans = ans._fix(context)+            return ans++        # Special case for multiplying by power of 10+        if self._int == '1':+            ans = _dec_from_triple(resultsign, other._int, resultexp)+            ans = ans._fix(context)+            return ans+        if other._int == '1':+            ans = _dec_from_triple(resultsign, self._int, resultexp)+            ans = ans._fix(context)+            return ans++        op1 = _WorkRep(self)+        op2 = _WorkRep(other)++        ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp)+        ans = ans._fix(context)++        return ans+    __rmul__ = __mul__++    def __truediv__(self, other, context=None):+        """Return self / other."""+        other = _convert_other(other)+        if other is NotImplemented:+            return NotImplemented++        if context is None:+            context = getcontext()++        sign = self._sign ^ other._sign++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context)+            if ans:+                return ans++            if self._isinfinity() and other._isinfinity():+                return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF')++            if self._isinfinity():+                return _SignedInfinity[sign]++            if other._isinfinity():+                context._raise_error(Clamped, 'Division by infinity')+                return _dec_from_triple(sign, '0', context.Etiny())++        # Special cases for zeroes+        if not other:+            if not self:+                return context._raise_error(DivisionUndefined, '0 / 0')+            return context._raise_error(DivisionByZero, 'x / 0', sign)++        if not self:+            exp = self._exp - other._exp+            coeff = 0+        else:+            # OK, so neither = 0, INF or NaN+            shift = len(other._int) - len(self._int) + context.prec + 1+            exp = self._exp - other._exp - shift+            op1 = _WorkRep(self)+            op2 = _WorkRep(other)+            if shift >= 0:+                coeff, remainder = divmod(op1.int * 10**shift, op2.int)+            else:+                coeff, remainder = divmod(op1.int, op2.int * 10**-shift)+            if remainder:+                # result is not exact; adjust to ensure correct rounding+                if coeff % 5 == 0:+                    coeff += 1+            else:+                # result is exact; get as close to ideal exponent as possible+                ideal_exp = self._exp - other._exp+                while exp < ideal_exp and coeff % 10 == 0:+                    coeff //= 10+                    exp += 1++        ans = _dec_from_triple(sign, str(coeff), exp)+        return ans._fix(context)++    def _divide(self, other, context):+        """Return (self // other, self % other), to context.prec precision.++        Assumes that neither self nor other is a NaN, that self is not+        infinite and that other is nonzero.+        """+        sign = self._sign ^ other._sign+        if other._isinfinity():+            ideal_exp = self._exp+        else:+            ideal_exp = min(self._exp, other._exp)++        expdiff = self.adjusted() - other.adjusted()+        if not self or other._isinfinity() or expdiff <= -2:+            return (_dec_from_triple(sign, '0', 0),+                    self._rescale(ideal_exp, context.rounding))+        if expdiff <= context.prec:+            op1 = _WorkRep(self)+            op2 = _WorkRep(other)+            if op1.exp >= op2.exp:+                op1.int *= 10**(op1.exp - op2.exp)+            else:+                op2.int *= 10**(op2.exp - op1.exp)+            q, r = divmod(op1.int, op2.int)+            if q < 10**context.prec:+                return (_dec_from_triple(sign, str(q), 0),+                        _dec_from_triple(self._sign, str(r), ideal_exp))++        # Here the quotient is too large to be representable+        ans = context._raise_error(DivisionImpossible,+                                   'quotient too large in //, % or divmod')+        return ans, ans++    def __rtruediv__(self, other, context=None):+        """Swaps self/other and returns __truediv__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__truediv__(self, context=context)++    def __divmod__(self, other, context=None):+        """+        Return (self // other, self % other)+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return (ans, ans)++        sign = self._sign ^ other._sign+        if self._isinfinity():+            if other._isinfinity():+                ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)')+                return ans, ans+            else:+                return (_SignedInfinity[sign],+                        context._raise_error(InvalidOperation, 'INF % x'))++        if not other:+            if not self:+                ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)')+                return ans, ans+            else:+                return (context._raise_error(DivisionByZero, 'x // 0', sign),+                        context._raise_error(InvalidOperation, 'x % 0'))++        quotient, remainder = self._divide(other, context)+        remainder = remainder._fix(context)+        return quotient, remainder++    def __rdivmod__(self, other, context=None):+        """Swaps self/other and returns __divmod__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__divmod__(self, context=context)++    def __mod__(self, other, context=None):+        """+        self % other+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if self._isinfinity():+            return context._raise_error(InvalidOperation, 'INF % x')+        elif not other:+            if self:+                return context._raise_error(InvalidOperation, 'x % 0')+            else:+                return context._raise_error(DivisionUndefined, '0 % 0')++        remainder = self._divide(other, context)[1]+        remainder = remainder._fix(context)+        return remainder++    def __rmod__(self, other, context=None):+        """Swaps self/other and returns __mod__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__mod__(self, context=context)++    def remainder_near(self, other, context=None):+        """+        Remainder nearest to 0-  abs(remainder-near) <= other/2+        """+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        # self == +/-infinity -> InvalidOperation+        if self._isinfinity():+            return context._raise_error(InvalidOperation,+                                        'remainder_near(infinity, x)')++        # other == 0 -> either InvalidOperation or DivisionUndefined+        if not other:+            if self:+                return context._raise_error(InvalidOperation,+                                            'remainder_near(x, 0)')+            else:+                return context._raise_error(DivisionUndefined,+                                            'remainder_near(0, 0)')++        # other = +/-infinity -> remainder = self+        if other._isinfinity():+            ans = Decimal(self)+            return ans._fix(context)++        # self = 0 -> remainder = self, with ideal exponent+        ideal_exponent = min(self._exp, other._exp)+        if not self:+            ans = _dec_from_triple(self._sign, '0', ideal_exponent)+            return ans._fix(context)++        # catch most cases of large or small quotient+        expdiff = self.adjusted() - other.adjusted()+        if expdiff >= context.prec + 1:+            # expdiff >= prec+1 => abs(self/other) > 10**prec+            return context._raise_error(DivisionImpossible)+        if expdiff <= -2:+            # expdiff <= -2 => abs(self/other) < 0.1+            ans = self._rescale(ideal_exponent, context.rounding)+            return ans._fix(context)++        # adjust both arguments to have the same exponent, then divide+        op1 = _WorkRep(self)+        op2 = _WorkRep(other)+        if op1.exp >= op2.exp:+            op1.int *= 10**(op1.exp - op2.exp)+        else:+            op2.int *= 10**(op2.exp - op1.exp)+        q, r = divmod(op1.int, op2.int)+        # remainder is r*10**ideal_exponent; other is +/-op2.int *+        # 10**ideal_exponent.   Apply correction to ensure that+        # abs(remainder) <= abs(other)/2+        if 2*r + (q&1) > op2.int:+            r -= op2.int+            q += 1++        if q >= 10**context.prec:+            return context._raise_error(DivisionImpossible)++        # result has same sign as self unless r is negative+        sign = self._sign+        if r < 0:+            sign = 1-sign+            r = -r++        ans = _dec_from_triple(sign, str(r), ideal_exponent)+        return ans._fix(context)++    def __floordiv__(self, other, context=None):+        """self // other"""+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if self._isinfinity():+            if other._isinfinity():+                return context._raise_error(InvalidOperation, 'INF // INF')+            else:+                return _SignedInfinity[self._sign ^ other._sign]++        if not other:+            if self:+                return context._raise_error(DivisionByZero, 'x // 0',+                                            self._sign ^ other._sign)+            else:+                return context._raise_error(DivisionUndefined, '0 // 0')++        return self._divide(other, context)[0]++    def __rfloordiv__(self, other, context=None):+        """Swaps self/other and returns __floordiv__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__floordiv__(self, context=context)++    def __float__(self):+        """Float representation."""+        if self._isnan():+            if self.is_snan():+                raise ValueError("Cannot convert signaling NaN to float")+            s = "-nan" if self._sign else "nan"+        else:+            s = str(self)+        return float(s)++    def __int__(self):+        """Converts self to an int, truncating if necessary."""+        if self._is_special:+            if self._isnan():+                raise ValueError("Cannot convert NaN to integer")+            elif self._isinfinity():+                raise OverflowError("Cannot convert infinity to integer")+        s = (-1)**self._sign+        if self._exp >= 0:+            return s*int(self._int)*10**self._exp+        else:+            return s*int(self._int[:self._exp] or '0')++    __trunc__ = __int__++    def real(self):+        return self+    real = property(real)++    def imag(self):+        return Decimal(0)+    imag = property(imag)++    def conjugate(self):+        return self++    def __complex__(self):+        return complex(float(self))++    def _fix_nan(self, context):+        """Decapitate the payload of a NaN to fit the context"""+        payload = self._int++        # maximum length of payload is precision if clamp=0,+        # precision-1 if clamp=1.+        max_payload_len = context.prec - context.clamp+        if len(payload) > max_payload_len:+            payload = payload[len(payload)-max_payload_len:].lstrip('0')+            return _dec_from_triple(self._sign, payload, self._exp, True)+        return Decimal(self)++    def _fix(self, context):+        """Round if it is necessary to keep self within prec precision.++        Rounds and fixes the exponent.  Does not raise on a sNaN.++        Arguments:+        self - Decimal instance+        context - context used.+        """++        if self._is_special:+            if self._isnan():+                # decapitate payload if necessary+                return self._fix_nan(context)+            else:+                # self is +/-Infinity; return unaltered+                return Decimal(self)++        # if self is zero then exponent should be between Etiny and+        # Emax if clamp==0, and between Etiny and Etop if clamp==1.+        Etiny = context.Etiny()+        Etop = context.Etop()+        if not self:+            exp_max = [context.Emax, Etop][context.clamp]+            new_exp = min(max(self._exp, Etiny), exp_max)+            if new_exp != self._exp:+                context._raise_error(Clamped)+                return _dec_from_triple(self._sign, '0', new_exp)+            else:+                return Decimal(self)++        # exp_min is the smallest allowable exponent of the result,+        # equal to max(self.adjusted()-context.prec+1, Etiny)+        exp_min = len(self._int) + self._exp - context.prec+        if exp_min > Etop:+            # overflow: exp_min > Etop iff self.adjusted() > Emax+            ans = context._raise_error(Overflow, 'above Emax', self._sign)+            context._raise_error(Inexact)+            context._raise_error(Rounded)+            return ans++        self_is_subnormal = exp_min < Etiny+        if self_is_subnormal:+            exp_min = Etiny++        # round if self has too many digits+        if self._exp < exp_min:+            digits = len(self._int) + self._exp - exp_min+            if digits < 0:+                self = _dec_from_triple(self._sign, '1', exp_min-1)+                digits = 0+            rounding_method = self._pick_rounding_function[context.rounding]+            changed = rounding_method(self, digits)+            coeff = self._int[:digits] or '0'+            if changed > 0:+                coeff = str(int(coeff)+1)+                if len(coeff) > context.prec:+                    coeff = coeff[:-1]+                    exp_min += 1++            # check whether the rounding pushed the exponent out of range+            if exp_min > Etop:+                ans = context._raise_error(Overflow, 'above Emax', self._sign)+            else:+                ans = _dec_from_triple(self._sign, coeff, exp_min)++            # raise the appropriate signals, taking care to respect+            # the precedence described in the specification+            if changed and self_is_subnormal:+                context._raise_error(Underflow)+            if self_is_subnormal:+                context._raise_error(Subnormal)+            if changed:+                context._raise_error(Inexact)+            context._raise_error(Rounded)+            if not ans:+                # raise Clamped on underflow to 0+                context._raise_error(Clamped)+            return ans++        if self_is_subnormal:+            context._raise_error(Subnormal)++        # fold down if clamp == 1 and self has too few digits+        if context.clamp == 1 and self._exp > Etop:+            context._raise_error(Clamped)+            self_padded = self._int + '0'*(self._exp - Etop)+            return _dec_from_triple(self._sign, self_padded, Etop)++        # here self was representable to begin with; return unchanged+        return Decimal(self)++    # for each of the rounding functions below:+    #   self is a finite, nonzero Decimal+    #   prec is an integer satisfying 0 <= prec < len(self._int)+    #+    # each function returns either -1, 0, or 1, as follows:+    #   1 indicates that self should be rounded up (away from zero)+    #   0 indicates that self should be truncated, and that all the+    #     digits to be truncated are zeros (so the value is unchanged)+    #  -1 indicates that there are nonzero digits to be truncated++    def _round_down(self, prec):+        """Also known as round-towards-0, truncate."""+        if _all_zeros(self._int, prec):+            return 0+        else:+            return -1++    def _round_up(self, prec):+        """Rounds away from 0."""+        return -self._round_down(prec)++    def _round_half_up(self, prec):+        """Rounds 5 up (away from 0)"""+        if self._int[prec] in '56789':+            return 1+        elif _all_zeros(self._int, prec):+            return 0+        else:+            return -1++    def _round_half_down(self, prec):+        """Round 5 down"""+        if _exact_half(self._int, prec):+            return -1+        else:+            return self._round_half_up(prec)++    def _round_half_even(self, prec):+        """Round 5 to even, rest to nearest."""+        if _exact_half(self._int, prec) and \+                (prec == 0 or self._int[prec-1] in '02468'):+            return -1+        else:+            return self._round_half_up(prec)++    def _round_ceiling(self, prec):+        """Rounds up (not away from 0 if negative.)"""+        if self._sign:+            return self._round_down(prec)+        else:+            return -self._round_down(prec)++    def _round_floor(self, prec):+        """Rounds down (not towards 0 if negative)"""+        if not self._sign:+            return self._round_down(prec)+        else:+            return -self._round_down(prec)++    def _round_05up(self, prec):+        """Round down unless digit prec-1 is 0 or 5."""+        if prec and self._int[prec-1] not in '05':+            return self._round_down(prec)+        else:+            return -self._round_down(prec)++    _pick_rounding_function = dict(+        ROUND_DOWN = _round_down,+        ROUND_UP = _round_up,+        ROUND_HALF_UP = _round_half_up,+        ROUND_HALF_DOWN = _round_half_down,+        ROUND_HALF_EVEN = _round_half_even,+        ROUND_CEILING = _round_ceiling,+        ROUND_FLOOR = _round_floor,+        ROUND_05UP = _round_05up,+    )++    def __round__(self, n=None):+        """Round self to the nearest integer, or to a given precision.++        If only one argument is supplied, round a finite Decimal+        instance self to the nearest integer.  If self is infinite or+        a NaN then a Python exception is raised.  If self is finite+        and lies exactly halfway between two integers then it is+        rounded to the integer with even last digit.++        >>> round(Decimal('123.456'))+        123+        >>> round(Decimal('-456.789'))+        -457+        >>> round(Decimal('-3.0'))+        -3+        >>> round(Decimal('2.5'))+        2+        >>> round(Decimal('3.5'))+        4+        >>> round(Decimal('Inf'))+        Traceback (most recent call last):+          ...+        OverflowError: cannot round an infinity+        >>> round(Decimal('NaN'))+        Traceback (most recent call last):+          ...+        ValueError: cannot round a NaN++        If a second argument n is supplied, self is rounded to n+        decimal places using the rounding mode for the current+        context.++        For an integer n, round(self, -n) is exactly equivalent to+        self.quantize(Decimal('1En')).++        >>> round(Decimal('123.456'), 0)+        Decimal('123')+        >>> round(Decimal('123.456'), 2)+        Decimal('123.46')+        >>> round(Decimal('123.456'), -2)+        Decimal('1E+2')+        >>> round(Decimal('-Infinity'), 37)+        Decimal('NaN')+        >>> round(Decimal('sNaN123'), 0)+        Decimal('NaN123')++        """+        if n is not None:+            # two-argument form: use the equivalent quantize call+            if not isinstance(n, int):+                raise TypeError('Second argument to round should be integral')+            exp = _dec_from_triple(0, '1', -n)+            return self.quantize(exp)++        # one-argument form+        if self._is_special:+            if self.is_nan():+                raise ValueError("cannot round a NaN")+            else:+                raise OverflowError("cannot round an infinity")+        return int(self._rescale(0, ROUND_HALF_EVEN))++    def __floor__(self):+        """Return the floor of self, as an integer.++        For a finite Decimal instance self, return the greatest+        integer n such that n <= self.  If self is infinite or a NaN+        then a Python exception is raised.++        """+        if self._is_special:+            if self.is_nan():+                raise ValueError("cannot round a NaN")+            else:+                raise OverflowError("cannot round an infinity")+        return int(self._rescale(0, ROUND_FLOOR))++    def __ceil__(self):+        """Return the ceiling of self, as an integer.++        For a finite Decimal instance self, return the least integer n+        such that n >= self.  If self is infinite or a NaN then a+        Python exception is raised.++        """+        if self._is_special:+            if self.is_nan():+                raise ValueError("cannot round a NaN")+            else:+                raise OverflowError("cannot round an infinity")+        return int(self._rescale(0, ROUND_CEILING))++    def fma(self, other, third, context=None):+        """Fused multiply-add.++        Returns self*other+third with no rounding of the intermediate+        product self*other.++        self and other are multiplied together, with no rounding of+        the result.  The third operand is then added to the result,+        and a single final rounding is performed.+        """++        other = _convert_other(other, raiseit=True)+        third = _convert_other(third, raiseit=True)++        # compute product; raise InvalidOperation if either operand is+        # a signaling NaN or if the product is zero times infinity.+        if self._is_special or other._is_special:+            if context is None:+                context = getcontext()+            if self._exp == 'N':+                return context._raise_error(InvalidOperation, 'sNaN', self)+            if other._exp == 'N':+                return context._raise_error(InvalidOperation, 'sNaN', other)+            if self._exp == 'n':+                product = self+            elif other._exp == 'n':+                product = other+            elif self._exp == 'F':+                if not other:+                    return context._raise_error(InvalidOperation,+                                                'INF * 0 in fma')+                product = _SignedInfinity[self._sign ^ other._sign]+            elif other._exp == 'F':+                if not self:+                    return context._raise_error(InvalidOperation,+                                                '0 * INF in fma')+                product = _SignedInfinity[self._sign ^ other._sign]+        else:+            product = _dec_from_triple(self._sign ^ other._sign,+                                       str(int(self._int) * int(other._int)),+                                       self._exp + other._exp)++        return product.__add__(third, context)++    def _power_modulo(self, other, modulo, context=None):+        """Three argument version of __pow__"""++        other = _convert_other(other)+        if other is NotImplemented:+            return other+        modulo = _convert_other(modulo)+        if modulo is NotImplemented:+            return modulo++        if context is None:+            context = getcontext()++        # deal with NaNs: if there are any sNaNs then first one wins,+        # (i.e. behaviour for NaNs is identical to that of fma)+        self_is_nan = self._isnan()+        other_is_nan = other._isnan()+        modulo_is_nan = modulo._isnan()+        if self_is_nan or other_is_nan or modulo_is_nan:+            if self_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        self)+            if other_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        other)+            if modulo_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        modulo)+            if self_is_nan:+                return self._fix_nan(context)+            if other_is_nan:+                return other._fix_nan(context)+            return modulo._fix_nan(context)++        # check inputs: we apply same restrictions as Python's pow()+        if not (self._isinteger() and+                other._isinteger() and+                modulo._isinteger()):+            return context._raise_error(InvalidOperation,+                                        'pow() 3rd argument not allowed '+                                        'unless all arguments are integers')+        if other < 0:+            return context._raise_error(InvalidOperation,+                                        'pow() 2nd argument cannot be '+                                        'negative when 3rd argument specified')+        if not modulo:+            return context._raise_error(InvalidOperation,+                                        'pow() 3rd argument cannot be 0')++        # additional restriction for decimal: the modulus must be less+        # than 10**prec in absolute value+        if modulo.adjusted() >= context.prec:+            return context._raise_error(InvalidOperation,+                                        'insufficient precision: pow() 3rd '+                                        'argument must not have more than '+                                        'precision digits')++        # define 0**0 == NaN, for consistency with two-argument pow+        # (even though it hurts!)+        if not other and not self:+            return context._raise_error(InvalidOperation,+                                        'at least one of pow() 1st argument '+                                        'and 2nd argument must be nonzero ;'+                                        '0**0 is not defined')++        # compute sign of result+        if other._iseven():+            sign = 0+        else:+            sign = self._sign++        # convert modulo to a Python integer, and self and other to+        # Decimal integers (i.e. force their exponents to be >= 0)+        modulo = abs(int(modulo))+        base = _WorkRep(self.to_integral_value())+        exponent = _WorkRep(other.to_integral_value())++        # compute result using integer pow()+        base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo+        for i in range(exponent.exp):+            base = pow(base, 10, modulo)+        base = pow(base, exponent.int, modulo)++        return _dec_from_triple(sign, str(base), 0)++    def _power_exact(self, other, p):+        """Attempt to compute self**other exactly.++        Given Decimals self and other and an integer p, attempt to+        compute an exact result for the power self**other, with p+        digits of precision.  Return None if self**other is not+        exactly representable in p digits.++        Assumes that elimination of special cases has already been+        performed: self and other must both be nonspecial; self must+        be positive and not numerically equal to 1; other must be+        nonzero.  For efficiency, other._exp should not be too large,+        so that 10**abs(other._exp) is a feasible calculation."""++        # In the comments below, we write x for the value of self and y for the+        # value of other.  Write x = xc*10**xe and abs(y) = yc*10**ye, with xc+        # and yc positive integers not divisible by 10.++        # The main purpose of this method is to identify the *failure*+        # of x**y to be exactly representable with as little effort as+        # possible.  So we look for cheap and easy tests that+        # eliminate the possibility of x**y being exact.  Only if all+        # these tests are passed do we go on to actually compute x**y.++        # Here's the main idea.  Express y as a rational number m/n, with m and+        # n relatively prime and n>0.  Then for x**y to be exactly+        # representable (at *any* precision), xc must be the nth power of a+        # positive integer and xe must be divisible by n.  If y is negative+        # then additionally xc must be a power of either 2 or 5, hence a power+        # of 2**n or 5**n.+        #+        # There's a limit to how small |y| can be: if y=m/n as above+        # then:+        #+        #  (1) if xc != 1 then for the result to be representable we+        #      need xc**(1/n) >= 2, and hence also xc**|y| >= 2.  So+        #      if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <=+        #      2**(1/|y|), hence xc**|y| < 2 and the result is not+        #      representable.+        #+        #  (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1.  Hence if+        #      |y| < 1/|xe| then the result is not representable.+        #+        # Note that since x is not equal to 1, at least one of (1) and+        # (2) must apply.  Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) <+        # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye.+        #+        # There's also a limit to how large y can be, at least if it's+        # positive: the normalized result will have coefficient xc**y,+        # so if it's representable then xc**y < 10**p, and y <+        # p/log10(xc).  Hence if y*log10(xc) >= p then the result is+        # not exactly representable.++        # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye,+        # so |y| < 1/xe and the result is not representable.+        # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y|+        # < 1/nbits(xc).++        x = _WorkRep(self)+        xc, xe = x.int, x.exp+        while xc % 10 == 0:+            xc //= 10+            xe += 1++        y = _WorkRep(other)+        yc, ye = y.int, y.exp+        while yc % 10 == 0:+            yc //= 10+            ye += 1++        # case where xc == 1: result is 10**(xe*y), with xe*y+        # required to be an integer+        if xc == 1:+            xe *= yc+            # result is now 10**(xe * 10**ye);  xe * 10**ye must be integral+            while xe % 10 == 0:+                xe //= 10+                ye += 1+            if ye < 0:+                return None+            exponent = xe * 10**ye+            if y.sign == 1:+                exponent = -exponent+            # if other is a nonnegative integer, use ideal exponent+            if other._isinteger() and other._sign == 0:+                ideal_exponent = self._exp*int(other)+                zeros = min(exponent-ideal_exponent, p-1)+            else:+                zeros = 0+            return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros)++        # case where y is negative: xc must be either a power+        # of 2 or a power of 5.+        if y.sign == 1:+            last_digit = xc % 10+            if last_digit in (2,4,6,8):+                # quick test for power of 2+                if xc & -xc != xc:+                    return None+                # now xc is a power of 2; e is its exponent+                e = _nbits(xc)-1++                # We now have:+                #+                #   x = 2**e * 10**xe, e > 0, and y < 0.+                #+                # The exact result is:+                #+                #   x**y = 5**(-e*y) * 10**(e*y + xe*y)+                #+                # provided that both e*y and xe*y are integers.  Note that if+                # 5**(-e*y) >= 10**p, then the result can't be expressed+                # exactly with p digits of precision.+                #+                # Using the above, we can guard against large values of ye.+                # 93/65 is an upper bound for log(10)/log(5), so if+                #+                #   ye >= len(str(93*p//65))+                #+                # then+                #+                #   -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5),+                #+                # so 5**(-e*y) >= 10**p, and the coefficient of the result+                # can't be expressed in p digits.++                # emax >= largest e such that 5**e < 10**p.+                emax = p*93//65+                if ye >= len(str(emax)):+                    return None++                # Find -e*y and -xe*y; both must be integers+                e = _decimal_lshift_exact(e * yc, ye)+                xe = _decimal_lshift_exact(xe * yc, ye)+                if e is None or xe is None:+                    return None++                if e > emax:+                    return None+                xc = 5**e++            elif last_digit == 5:+                # e >= log_5(xc) if xc is a power of 5; we have+                # equality all the way up to xc=5**2658+                e = _nbits(xc)*28//65+                xc, remainder = divmod(5**e, xc)+                if remainder:+                    return None+                while xc % 5 == 0:+                    xc //= 5+                    e -= 1++                # Guard against large values of ye, using the same logic as in+                # the 'xc is a power of 2' branch.  10/3 is an upper bound for+                # log(10)/log(2).+                emax = p*10//3+                if ye >= len(str(emax)):+                    return None++                e = _decimal_lshift_exact(e * yc, ye)+                xe = _decimal_lshift_exact(xe * yc, ye)+                if e is None or xe is None:+                    return None++                if e > emax:+                    return None+                xc = 2**e+            else:+                return None++            if xc >= 10**p:+                return None+            xe = -e-xe+            return _dec_from_triple(0, str(xc), xe)++        # now y is positive; find m and n such that y = m/n+        if ye >= 0:+            m, n = yc*10**ye, 1+        else:+            if xe != 0 and len(str(abs(yc*xe))) <= -ye:+                return None+            xc_bits = _nbits(xc)+            if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:+                return None+            m, n = yc, 10**(-ye)+            while m % 2 == n % 2 == 0:+                m //= 2+                n //= 2+            while m % 5 == n % 5 == 0:+                m //= 5+                n //= 5++        # compute nth root of xc*10**xe+        if n > 1:+            # if 1 < xc < 2**n then xc isn't an nth power+            if xc != 1 and xc_bits <= n:+                return None++            xe, rem = divmod(xe, n)+            if rem != 0:+                return None++            # compute nth root of xc using Newton's method+            a = 1 << -(-_nbits(xc)//n) # initial estimate+            while True:+                q, r = divmod(xc, a**(n-1))+                if a <= q:+                    break+                else:+                    a = (a*(n-1) + q)//n+            if not (a == q and r == 0):+                return None+            xc = a++        # now xc*10**xe is the nth root of the original xc*10**xe+        # compute mth power of xc*10**xe++        # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m >+        # 10**p and the result is not representable.+        if xc > 1 and m > p*100//_log10_lb(xc):+            return None+        xc = xc**m+        xe *= m+        if xc > 10**p:+            return None++        # by this point the result *is* exactly representable+        # adjust the exponent to get as close as possible to the ideal+        # exponent, if necessary+        str_xc = str(xc)+        if other._isinteger() and other._sign == 0:+            ideal_exponent = self._exp*int(other)+            zeros = min(xe-ideal_exponent, p-len(str_xc))+        else:+            zeros = 0+        return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros)++    def __pow__(self, other, modulo=None, context=None):+        """Return self ** other [ % modulo].++        With two arguments, compute self**other.++        With three arguments, compute (self**other) % modulo.  For the+        three argument form, the following restrictions on the+        arguments hold:++         - all three arguments must be integral+         - other must be nonnegative+         - either self or other (or both) must be nonzero+         - modulo must be nonzero and must have at most p digits,+           where p is the context precision.++        If any of these restrictions is violated the InvalidOperation+        flag is raised.++        The result of pow(self, other, modulo) is identical to the+        result that would be obtained by computing (self**other) %+        modulo with unbounded precision, but is computed more+        efficiently.  It is always exact.+        """++        if modulo is not None:+            return self._power_modulo(other, modulo, context)++        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        # either argument is a NaN => result is NaN+        ans = self._check_nans(other, context)+        if ans:+            return ans++        # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity)+        if not other:+            if not self:+                return context._raise_error(InvalidOperation, '0 ** 0')+            else:+                return _One++        # result has sign 1 iff self._sign is 1 and other is an odd integer+        result_sign = 0+        if self._sign == 1:+            if other._isinteger():+                if not other._iseven():+                    result_sign = 1+            else:+                # -ve**noninteger = NaN+                # (-0)**noninteger = 0**noninteger+                if self:+                    return context._raise_error(InvalidOperation,+                        'x ** y with x negative and y not an integer')+            # negate self, without doing any unwanted rounding+            self = self.copy_negate()++        # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity+        if not self:+            if other._sign == 0:+                return _dec_from_triple(result_sign, '0', 0)+            else:+                return _SignedInfinity[result_sign]++        # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0+        if self._isinfinity():+            if other._sign == 0:+                return _SignedInfinity[result_sign]+            else:+                return _dec_from_triple(result_sign, '0', 0)++        # 1**other = 1, but the choice of exponent and the flags+        # depend on the exponent of self, and on whether other is a+        # positive integer, a negative integer, or neither+        if self == _One:+            if other._isinteger():+                # exp = max(self._exp*max(int(other), 0),+                # 1-context.prec) but evaluating int(other) directly+                # is dangerous until we know other is small (other+                # could be 1e999999999)+                if other._sign == 1:+                    multiplier = 0+                elif other > context.prec:+                    multiplier = context.prec+                else:+                    multiplier = int(other)++                exp = self._exp * multiplier+                if exp < 1-context.prec:+                    exp = 1-context.prec+                    context._raise_error(Rounded)+            else:+                context._raise_error(Inexact)+                context._raise_error(Rounded)+                exp = 1-context.prec++            return _dec_from_triple(result_sign, '1'+'0'*-exp, exp)++        # compute adjusted exponent of self+        self_adj = self.adjusted()++        # self ** infinity is infinity if self > 1, 0 if self < 1+        # self ** -infinity is infinity if self < 1, 0 if self > 1+        if other._isinfinity():+            if (other._sign == 0) == (self_adj < 0):+                return _dec_from_triple(result_sign, '0', 0)+            else:+                return _SignedInfinity[result_sign]++        # from here on, the result always goes through the call+        # to _fix at the end of this function.+        ans = None+        exact = False++        # crude test to catch cases of extreme overflow/underflow.  If+        # log10(self)*other >= 10**bound and bound >= len(str(Emax))+        # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence+        # self**other >= 10**(Emax+1), so overflow occurs.  The test+        # for underflow is similar.+        bound = self._log10_exp_bound() + other.adjusted()+        if (self_adj >= 0) == (other._sign == 0):+            # self > 1 and other +ve, or self < 1 and other -ve+            # possibility of overflow+            if bound >= len(str(context.Emax)):+                ans = _dec_from_triple(result_sign, '1', context.Emax+1)+        else:+            # self > 1 and other -ve, or self < 1 and other +ve+            # possibility of underflow to 0+            Etiny = context.Etiny()+            if bound >= len(str(-Etiny)):+                ans = _dec_from_triple(result_sign, '1', Etiny-1)++        # try for an exact result with precision +1+        if ans is None:+            ans = self._power_exact(other, context.prec + 1)+            if ans is not None:+                if result_sign == 1:+                    ans = _dec_from_triple(1, ans._int, ans._exp)+                exact = True++        # usual case: inexact result, x**y computed directly as exp(y*log(x))+        if ans is None:+            p = context.prec+            x = _WorkRep(self)+            xc, xe = x.int, x.exp+            y = _WorkRep(other)+            yc, ye = y.int, y.exp+            if y.sign == 1:+                yc = -yc++            # compute correctly rounded result:  start with precision +3,+            # then increase precision until result is unambiguously roundable+            extra = 3+            while True:+                coeff, exp = _dpower(xc, xe, yc, ye, p+extra)+                if coeff % (5*10**(len(str(coeff))-p-1)):+                    break+                extra += 3++            ans = _dec_from_triple(result_sign, str(coeff), exp)++        # unlike exp, ln and log10, the power function respects the+        # rounding mode; no need to switch to ROUND_HALF_EVEN here++        # There's a difficulty here when 'other' is not an integer and+        # the result is exact.  In this case, the specification+        # requires that the Inexact flag be raised (in spite of+        # exactness), but since the result is exact _fix won't do this+        # for us.  (Correspondingly, the Underflow signal should also+        # be raised for subnormal results.)  We can't directly raise+        # these signals either before or after calling _fix, since+        # that would violate the precedence for signals.  So we wrap+        # the ._fix call in a temporary context, and reraise+        # afterwards.+        if exact and not other._isinteger():+            # pad with zeros up to length context.prec+1 if necessary; this+            # ensures that the Rounded signal will be raised.+            if len(ans._int) <= context.prec:+                expdiff = context.prec + 1 - len(ans._int)+                ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff,+                                       ans._exp-expdiff)++            # create a copy of the current context, with cleared flags/traps+            newcontext = context.copy()+            newcontext.clear_flags()+            for exception in _signals:+                newcontext.traps[exception] = 0++            # round in the new context+            ans = ans._fix(newcontext)++            # raise Inexact, and if necessary, Underflow+            newcontext._raise_error(Inexact)+            if newcontext.flags[Subnormal]:+                newcontext._raise_error(Underflow)++            # propagate signals to the original context; _fix could+            # have raised any of Overflow, Underflow, Subnormal,+            # Inexact, Rounded, Clamped.  Overflow needs the correct+            # arguments.  Note that the order of the exceptions is+            # important here.+            if newcontext.flags[Overflow]:+                context._raise_error(Overflow, 'above Emax', ans._sign)+            for exception in Underflow, Subnormal, Inexact, Rounded, Clamped:+                if newcontext.flags[exception]:+                    context._raise_error(exception)++        else:+            ans = ans._fix(context)++        return ans++    def __rpow__(self, other, context=None):+        """Swaps self/other and returns __pow__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__pow__(self, context=context)++    def normalize(self, context=None):+        """Normalize- strip trailing 0s, change anything equal to 0 to 0e0"""++        if context is None:+            context = getcontext()++        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        dup = self._fix(context)+        if dup._isinfinity():+            return dup++        if not dup:+            return _dec_from_triple(dup._sign, '0', 0)+        exp_max = [context.Emax, context.Etop()][context.clamp]+        end = len(dup._int)+        exp = dup._exp+        while dup._int[end-1] == '0' and exp < exp_max:+            exp += 1+            end -= 1+        return _dec_from_triple(dup._sign, dup._int[:end], exp)++    def quantize(self, exp, rounding=None, context=None):+        """Quantize self so its exponent is the same as that of exp.++        Similar to self._rescale(exp._exp) but with error checking.+        """+        exp = _convert_other(exp, raiseit=True)++        if context is None:+            context = getcontext()+        if rounding is None:+            rounding = context.rounding++        if self._is_special or exp._is_special:+            ans = self._check_nans(exp, context)+            if ans:+                return ans++            if exp._isinfinity() or self._isinfinity():+                if exp._isinfinity() and self._isinfinity():+                    return Decimal(self)  # if both are inf, it is OK+                return context._raise_error(InvalidOperation,+                                        'quantize with one INF')++        # exp._exp should be between Etiny and Emax+        if not (context.Etiny() <= exp._exp <= context.Emax):+            return context._raise_error(InvalidOperation,+                   'target exponent out of bounds in quantize')++        if not self:+            ans = _dec_from_triple(self._sign, '0', exp._exp)+            return ans._fix(context)++        self_adjusted = self.adjusted()+        if self_adjusted > context.Emax:+            return context._raise_error(InvalidOperation,+                                        'exponent of quantize result too large for current context')+        if self_adjusted - exp._exp + 1 > context.prec:+            return context._raise_error(InvalidOperation,+                                        'quantize result has too many digits for current context')++        ans = self._rescale(exp._exp, rounding)+        if ans.adjusted() > context.Emax:+            return context._raise_error(InvalidOperation,+                                        'exponent of quantize result too large for current context')+        if len(ans._int) > context.prec:+            return context._raise_error(InvalidOperation,+                                        'quantize result has too many digits for current context')++        # raise appropriate flags+        if ans and ans.adjusted() < context.Emin:+            context._raise_error(Subnormal)+        if ans._exp > self._exp:+            if ans != self:+                context._raise_error(Inexact)+            context._raise_error(Rounded)++        # call to fix takes care of any necessary folddown, and+        # signals Clamped if necessary+        ans = ans._fix(context)+        return ans++    def same_quantum(self, other, context=None):+        """Return True if self and other have the same exponent; otherwise+        return False.++        If either operand is a special value, the following rules are used:+           * return True if both operands are infinities+           * return True if both operands are NaNs+           * otherwise, return False.+        """+        other = _convert_other(other, raiseit=True)+        if self._is_special or other._is_special:+            return (self.is_nan() and other.is_nan() or+                    self.is_infinite() and other.is_infinite())+        return self._exp == other._exp++    def _rescale(self, exp, rounding):+        """Rescale self so that the exponent is exp, either by padding with zeros+        or by truncating digits, using the given rounding mode.++        Specials are returned without change.  This operation is+        quiet: it raises no flags, and uses no information from the+        context.++        exp = exp to scale to (an integer)+        rounding = rounding mode+        """+        if self._is_special:+            return Decimal(self)+        if not self:+            return _dec_from_triple(self._sign, '0', exp)++        if self._exp >= exp:+            # pad answer with zeros if necessary+            return _dec_from_triple(self._sign,+                                        self._int + '0'*(self._exp - exp), exp)++        # too many digits; round and lose data.  If self.adjusted() <+        # exp-1, replace self by 10**(exp-1) before rounding+        digits = len(self._int) + self._exp - exp+        if digits < 0:+            self = _dec_from_triple(self._sign, '1', exp-1)+            digits = 0+        this_function = self._pick_rounding_function[rounding]+        changed = this_function(self, digits)+        coeff = self._int[:digits] or '0'+        if changed == 1:+            coeff = str(int(coeff)+1)+        return _dec_from_triple(self._sign, coeff, exp)++    def _round(self, places, rounding):+        """Round a nonzero, nonspecial Decimal to a fixed number of+        significant figures, using the given rounding mode.++        Infinities, NaNs and zeros are returned unaltered.++        This operation is quiet: it raises no flags, and uses no+        information from the context.++        """+        if places <= 0:+            raise ValueError("argument should be at least 1 in _round")+        if self._is_special or not self:+            return Decimal(self)+        ans = self._rescale(self.adjusted()+1-places, rounding)+        # it can happen that the rescale alters the adjusted exponent;+        # for example when rounding 99.97 to 3 significant figures.+        # When this happens we end up with an extra 0 at the end of+        # the number; a second rescale fixes this.+        if ans.adjusted() != self.adjusted():+            ans = ans._rescale(ans.adjusted()+1-places, rounding)+        return ans++    def to_integral_exact(self, rounding=None, context=None):+        """Rounds to a nearby integer.++        If no rounding mode is specified, take the rounding mode from+        the context.  This method raises the Rounded and Inexact flags+        when appropriate.++        See also: to_integral_value, which does exactly the same as+        this method except that it doesn't raise Inexact or Rounded.+        """+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans+            return Decimal(self)+        if self._exp >= 0:+            return Decimal(self)+        if not self:+            return _dec_from_triple(self._sign, '0', 0)+        if context is None:+            context = getcontext()+        if rounding is None:+            rounding = context.rounding+        ans = self._rescale(0, rounding)+        if ans != self:+            context._raise_error(Inexact)+        context._raise_error(Rounded)+        return ans++    def to_integral_value(self, rounding=None, context=None):+        """Rounds to the nearest integer, without raising inexact, rounded."""+        if context is None:+            context = getcontext()+        if rounding is None:+            rounding = context.rounding+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans+            return Decimal(self)+        if self._exp >= 0:+            return Decimal(self)+        else:+            return self._rescale(0, rounding)++    # the method name changed, but we provide also the old one, for compatibility+    to_integral = to_integral_value++    def sqrt(self, context=None):+        """Return the square root of self."""+        if context is None:+            context = getcontext()++        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++            if self._isinfinity() and self._sign == 0:+                return Decimal(self)++        if not self:+            # exponent = self._exp // 2.  sqrt(-0) = -0+            ans = _dec_from_triple(self._sign, '0', self._exp // 2)+            return ans._fix(context)++        if self._sign == 1:+            return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0')++        # At this point self represents a positive number.  Let p be+        # the desired precision and express self in the form c*100**e+        # with c a positive real number and e an integer, c and e+        # being chosen so that 100**(p-1) <= c < 100**p.  Then the+        # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1)+        # <= sqrt(c) < 10**p, so the closest representable Decimal at+        # precision p is n*10**e where n = round_half_even(sqrt(c)),+        # the closest integer to sqrt(c) with the even integer chosen+        # in the case of a tie.+        #+        # To ensure correct rounding in all cases, we use the+        # following trick: we compute the square root to an extra+        # place (precision p+1 instead of precision p), rounding down.+        # Then, if the result is inexact and its last digit is 0 or 5,+        # we increase the last digit to 1 or 6 respectively; if it's+        # exact we leave the last digit alone.  Now the final round to+        # p places (or fewer in the case of underflow) will round+        # correctly and raise the appropriate flags.++        # use an extra digit of precision+        prec = context.prec+1++        # write argument in the form c*100**e where e = self._exp//2+        # is the 'ideal' exponent, to be used if the square root is+        # exactly representable.  l is the number of 'digits' of c in+        # base 100, so that 100**(l-1) <= c < 100**l.+        op = _WorkRep(self)+        e = op.exp >> 1+        if op.exp & 1:+            c = op.int * 10+            l = (len(self._int) >> 1) + 1+        else:+            c = op.int+            l = len(self._int)+1 >> 1++        # rescale so that c has exactly prec base 100 'digits'+        shift = prec-l+        if shift >= 0:+            c *= 100**shift+            exact = True+        else:+            c, remainder = divmod(c, 100**-shift)+            exact = not remainder+        e -= shift++        # find n = floor(sqrt(c)) using Newton's method+        n = 10**prec+        while True:+            q = c//n+            if n <= q:+                break+            else:+                n = n + q >> 1+        exact = exact and n*n == c++        if exact:+            # result is exact; rescale to use ideal exponent e+            if shift >= 0:+                # assert n % 10**shift == 0+                n //= 10**shift+            else:+                n *= 10**-shift+            e += shift+        else:+            # result is not exact; fix last digit as described above+            if n % 5 == 0:+                n += 1++        ans = _dec_from_triple(0, str(n), e)++        # round, and fit to current context+        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding++        return ans++    def max(self, other, context=None):+        """Returns the larger value.++        Like max(self, other) except if one is not a number, returns+        NaN (and signals if one is sNaN).  Also rounds.+        """+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self._cmp(other)+        if c == 0:+            # If both operands are finite and equal in numerical value+            # then an ordering is applied:+            #+            # If the signs differ then max returns the operand with the+            # positive sign and min returns the operand with the negative sign+            #+            # If the signs are the same then the exponent is used to select+            # the result.  This is exactly the ordering used in compare_total.+            c = self.compare_total(other)++        if c == -1:+            ans = other+        else:+            ans = self++        return ans._fix(context)++    def min(self, other, context=None):+        """Returns the smaller value.++        Like min(self, other) except if one is not a number, returns+        NaN (and signals if one is sNaN).  Also rounds.+        """+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self._cmp(other)+        if c == 0:+            c = self.compare_total(other)++        if c == -1:+            ans = self+        else:+            ans = other++        return ans._fix(context)++    def _isinteger(self):+        """Returns whether self is an integer"""+        if self._is_special:+            return False+        if self._exp >= 0:+            return True+        rest = self._int[self._exp:]+        return rest == '0'*len(rest)++    def _iseven(self):+        """Returns True if self is even.  Assumes self is an integer."""+        if not self or self._exp > 0:+            return True+        return self._int[-1+self._exp] in '02468'++    def adjusted(self):+        """Return the adjusted exponent of self"""+        try:+            return self._exp + len(self._int) - 1+        # If NaN or Infinity, self._exp is string+        except TypeError:+            return 0++    def canonical(self):+        """Returns the same Decimal object.++        As we do not have different encodings for the same number, the+        received object already is in its canonical form.+        """+        return self++    def compare_signal(self, other, context=None):+        """Compares self to the other operand numerically.++        It's pretty much like compare(), but all NaNs signal, with signaling+        NaNs taking precedence over quiet NaNs.+        """+        other = _convert_other(other, raiseit = True)+        ans = self._compare_check_nans(other, context)+        if ans:+            return ans+        return self.compare(other, context=context)++    def compare_total(self, other, context=None):+        """Compares self to other using the abstract representations.++        This is not like the standard compare, which use their numerical+        value. Note that a total ordering is defined for all possible abstract+        representations.+        """+        other = _convert_other(other, raiseit=True)++        # if one is negative and the other is positive, it's easy+        if self._sign and not other._sign:+            return _NegativeOne+        if not self._sign and other._sign:+            return _One+        sign = self._sign++        # let's handle both NaN types+        self_nan = self._isnan()+        other_nan = other._isnan()+        if self_nan or other_nan:+            if self_nan == other_nan:+                # compare payloads as though they're integers+                self_key = len(self._int), self._int+                other_key = len(other._int), other._int+                if self_key < other_key:+                    if sign:+                        return _One+                    else:+                        return _NegativeOne+                if self_key > other_key:+                    if sign:+                        return _NegativeOne+                    else:+                        return _One+                return _Zero++            if sign:+                if self_nan == 1:+                    return _NegativeOne+                if other_nan == 1:+                    return _One+                if self_nan == 2:+                    return _NegativeOne+                if other_nan == 2:+                    return _One+            else:+                if self_nan == 1:+                    return _One+                if other_nan == 1:+                    return _NegativeOne+                if self_nan == 2:+                    return _One+                if other_nan == 2:+                    return _NegativeOne++        if self < other:+            return _NegativeOne+        if self > other:+            return _One++        if self._exp < other._exp:+            if sign:+                return _One+            else:+                return _NegativeOne+        if self._exp > other._exp:+            if sign:+                return _NegativeOne+            else:+                return _One+        return _Zero+++    def compare_total_mag(self, other, context=None):+        """Compares self to other using abstract repr., ignoring sign.++        Like compare_total, but with operand's sign ignored and assumed to be 0.+        """+        other = _convert_other(other, raiseit=True)++        s = self.copy_abs()+        o = other.copy_abs()+        return s.compare_total(o)++    def copy_abs(self):+        """Returns a copy with the sign set to 0. """+        return _dec_from_triple(0, self._int, self._exp, self._is_special)++    def copy_negate(self):+        """Returns a copy with the sign inverted."""+        if self._sign:+            return _dec_from_triple(0, self._int, self._exp, self._is_special)+        else:+            return _dec_from_triple(1, self._int, self._exp, self._is_special)++    def copy_sign(self, other, context=None):+        """Returns self with the sign of other."""+        other = _convert_other(other, raiseit=True)+        return _dec_from_triple(other._sign, self._int,+                                self._exp, self._is_special)++    def exp(self, context=None):+        """Returns e ** self."""++        if context is None:+            context = getcontext()++        # exp(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        # exp(-Infinity) = 0+        if self._isinfinity() == -1:+            return _Zero++        # exp(0) = 1+        if not self:+            return _One++        # exp(Infinity) = Infinity+        if self._isinfinity() == 1:+            return Decimal(self)++        # the result is now guaranteed to be inexact (the true+        # mathematical result is transcendental). There's no need to+        # raise Rounded and Inexact here---they'll always be raised as+        # a result of the call to _fix.+        p = context.prec+        adj = self.adjusted()++        # we only need to do any computation for quite a small range+        # of adjusted exponents---for example, -29 <= adj <= 10 for+        # the default context.  For smaller exponent the result is+        # indistinguishable from 1 at the given precision, while for+        # larger exponent the result either overflows or underflows.+        if self._sign == 0 and adj > len(str((context.Emax+1)*3)):+            # overflow+            ans = _dec_from_triple(0, '1', context.Emax+1)+        elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)):+            # underflow to 0+            ans = _dec_from_triple(0, '1', context.Etiny()-1)+        elif self._sign == 0 and adj < -p:+            # p+1 digits; final round will raise correct flags+            ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p)+        elif self._sign == 1 and adj < -p-1:+            # p+1 digits; final round will raise correct flags+            ans = _dec_from_triple(0, '9'*(p+1), -p-1)+        # general case+        else:+            op = _WorkRep(self)+            c, e = op.int, op.exp+            if op.sign == 1:+                c = -c++            # compute correctly rounded result: increase precision by+            # 3 digits at a time until we get an unambiguously+            # roundable result+            extra = 3+            while True:+                coeff, exp = _dexp(c, e, p+extra)+                if coeff % (5*10**(len(str(coeff))-p-1)):+                    break+                extra += 3++            ans = _dec_from_triple(0, str(coeff), exp)++        # at this stage, ans should round correctly with *any*+        # rounding mode, not just with ROUND_HALF_EVEN+        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding++        return ans++    def is_canonical(self):+        """Return True if self is canonical; otherwise return False.++        Currently, the encoding of a Decimal instance is always+        canonical, so this method returns True for any Decimal.+        """+        return True++    def is_finite(self):+        """Return True if self is finite; otherwise return False.++        A Decimal instance is considered finite if it is neither+        infinite nor a NaN.+        """+        return not self._is_special++    def is_infinite(self):+        """Return True if self is infinite; otherwise return False."""+        return self._exp == 'F'++    def is_nan(self):+        """Return True if self is a qNaN or sNaN; otherwise return False."""+        return self._exp in ('n', 'N')++    def is_normal(self, context=None):+        """Return True if self is a normal number; otherwise return False."""+        if self._is_special or not self:+            return False+        if context is None:+            context = getcontext()+        return context.Emin <= self.adjusted()++    def is_qnan(self):+        """Return True if self is a quiet NaN; otherwise return False."""+        return self._exp == 'n'++    def is_signed(self):+        """Return True if self is negative; otherwise return False."""+        return self._sign == 1++    def is_snan(self):+        """Return True if self is a signaling NaN; otherwise return False."""+        return self._exp == 'N'++    def is_subnormal(self, context=None):+        """Return True if self is subnormal; otherwise return False."""+        if self._is_special or not self:+            return False+        if context is None:+            context = getcontext()+        return self.adjusted() < context.Emin++    def is_zero(self):+        """Return True if self is a zero; otherwise return False."""+        return not self._is_special and self._int == '0'++    def _ln_exp_bound(self):+        """Compute a lower bound for the adjusted exponent of self.ln().+        In other words, compute r such that self.ln() >= 10**r.  Assumes+        that self is finite and positive and that self != 1.+        """++        # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1+        adj = self._exp + len(self._int) - 1+        if adj >= 1:+            # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10)+            return len(str(adj*23//10)) - 1+        if adj <= -2:+            # argument <= 0.1+            return len(str((-1-adj)*23//10)) - 1+        op = _WorkRep(self)+        c, e = op.int, op.exp+        if adj == 0:+            # 1 < self < 10+            num = str(c-10**-e)+            den = str(c)+            return len(num) - len(den) - (num < den)+        # adj == -1, 0.1 <= self < 1+        return e + len(str(10**-e - c)) - 1+++    def ln(self, context=None):+        """Returns the natural (base e) logarithm of self."""++        if context is None:+            context = getcontext()++        # ln(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        # ln(0.0) == -Infinity+        if not self:+            return _NegativeInfinity++        # ln(Infinity) = Infinity+        if self._isinfinity() == 1:+            return _Infinity++        # ln(1.0) == 0.0+        if self == _One:+            return _Zero++        # ln(negative) raises InvalidOperation+        if self._sign == 1:+            return context._raise_error(InvalidOperation,+                                        'ln of a negative value')++        # result is irrational, so necessarily inexact+        op = _WorkRep(self)+        c, e = op.int, op.exp+        p = context.prec++        # correctly rounded result: repeatedly increase precision by 3+        # until we get an unambiguously roundable result+        places = p - self._ln_exp_bound() + 2 # at least p+3 places+        while True:+            coeff = _dlog(c, e, places)+            # assert len(str(abs(coeff)))-p >= 1+            if coeff % (5*10**(len(str(abs(coeff)))-p-1)):+                break+            places += 3+        ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)++        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding+        return ans++    def _log10_exp_bound(self):+        """Compute a lower bound for the adjusted exponent of self.log10().+        In other words, find r such that self.log10() >= 10**r.+        Assumes that self is finite and positive and that self != 1.+        """++        # For x >= 10 or x < 0.1 we only need a bound on the integer+        # part of log10(self), and this comes directly from the+        # exponent of x.  For 0.1 <= x <= 10 we use the inequalities+        # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| >+        # (1-1/x)/2.31 > 0.  If x < 1 then |log10(x)| > (1-x)/2.31 > 0++        adj = self._exp + len(self._int) - 1+        if adj >= 1:+            # self >= 10+            return len(str(adj))-1+        if adj <= -2:+            # self < 0.1+            return len(str(-1-adj))-1+        op = _WorkRep(self)+        c, e = op.int, op.exp+        if adj == 0:+            # 1 < self < 10+            num = str(c-10**-e)+            den = str(231*c)+            return len(num) - len(den) - (num < den) + 2+        # adj == -1, 0.1 <= self < 1+        num = str(10**-e-c)+        return len(num) + e - (num < "231") - 1++    def log10(self, context=None):+        """Returns the base 10 logarithm of self."""++        if context is None:+            context = getcontext()++        # log10(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        # log10(0.0) == -Infinity+        if not self:+            return _NegativeInfinity++        # log10(Infinity) = Infinity+        if self._isinfinity() == 1:+            return _Infinity++        # log10(negative or -Infinity) raises InvalidOperation+        if self._sign == 1:+            return context._raise_error(InvalidOperation,+                                        'log10 of a negative value')++        # log10(10**n) = n+        if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1):+            # answer may need rounding+            ans = Decimal(self._exp + len(self._int) - 1)+        else:+            # result is irrational, so necessarily inexact+            op = _WorkRep(self)+            c, e = op.int, op.exp+            p = context.prec++            # correctly rounded result: repeatedly increase precision+            # until result is unambiguously roundable+            places = p-self._log10_exp_bound()+2+            while True:+                coeff = _dlog10(c, e, places)+                # assert len(str(abs(coeff)))-p >= 1+                if coeff % (5*10**(len(str(abs(coeff)))-p-1)):+                    break+                places += 3+            ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)++        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding+        return ans++    def logb(self, context=None):+        """ Returns the exponent of the magnitude of self's MSD.++        The result is the integer which is the exponent of the magnitude+        of the most significant digit of self (as though it were truncated+        to a single digit while maintaining the value of that digit and+        without limiting the resulting exponent).+        """+        # logb(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        if context is None:+            context = getcontext()++        # logb(+/-Inf) = +Inf+        if self._isinfinity():+            return _Infinity++        # logb(0) = -Inf, DivisionByZero+        if not self:+            return context._raise_error(DivisionByZero, 'logb(0)', 1)++        # otherwise, simply return the adjusted exponent of self, as a+        # Decimal.  Note that no attempt is made to fit the result+        # into the current context.+        ans = Decimal(self.adjusted())+        return ans._fix(context)++    def _islogical(self):+        """Return True if self is a logical operand.++        For being logical, it must be a finite number with a sign of 0,+        an exponent of 0, and a coefficient whose digits must all be+        either 0 or 1.+        """+        if self._sign != 0 or self._exp != 0:+            return False+        for dig in self._int:+            if dig not in '01':+                return False+        return True++    def _fill_logical(self, context, opa, opb):+        dif = context.prec - len(opa)+        if dif > 0:+            opa = '0'*dif + opa+        elif dif < 0:+            opa = opa[-context.prec:]+        dif = context.prec - len(opb)+        if dif > 0:+            opb = '0'*dif + opb+        elif dif < 0:+            opb = opb[-context.prec:]+        return opa, opb++    def logical_and(self, other, context=None):+        """Applies an 'and' operation between self and other's digits."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        if not self._islogical() or not other._islogical():+            return context._raise_error(InvalidOperation)++        # fill to context.prec+        (opa, opb) = self._fill_logical(context, self._int, other._int)++        # make the operation, and clean starting zeroes+        result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)])+        return _dec_from_triple(0, result.lstrip('0') or '0', 0)++    def logical_invert(self, context=None):+        """Invert all its digits."""+        if context is None:+            context = getcontext()+        return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0),+                                context)++    def logical_or(self, other, context=None):+        """Applies an 'or' operation between self and other's digits."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        if not self._islogical() or not other._islogical():+            return context._raise_error(InvalidOperation)++        # fill to context.prec+        (opa, opb) = self._fill_logical(context, self._int, other._int)++        # make the operation, and clean starting zeroes+        result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)])+        return _dec_from_triple(0, result.lstrip('0') or '0', 0)++    def logical_xor(self, other, context=None):+        """Applies an 'xor' operation between self and other's digits."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        if not self._islogical() or not other._islogical():+            return context._raise_error(InvalidOperation)++        # fill to context.prec+        (opa, opb) = self._fill_logical(context, self._int, other._int)++        # make the operation, and clean starting zeroes+        result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)])+        return _dec_from_triple(0, result.lstrip('0') or '0', 0)++    def max_mag(self, other, context=None):+        """Compares the values numerically with their sign ignored."""+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self.copy_abs()._cmp(other.copy_abs())+        if c == 0:+            c = self.compare_total(other)++        if c == -1:+            ans = other+        else:+            ans = self++        return ans._fix(context)++    def min_mag(self, other, context=None):+        """Compares the values numerically with their sign ignored."""+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self.copy_abs()._cmp(other.copy_abs())+        if c == 0:+            c = self.compare_total(other)++        if c == -1:+            ans = self+        else:+            ans = other++        return ans._fix(context)++    def next_minus(self, context=None):+        """Returns the largest representable number smaller than itself."""+        if context is None:+            context = getcontext()++        ans = self._check_nans(context=context)+        if ans:+            return ans++        if self._isinfinity() == -1:+            return _NegativeInfinity+        if self._isinfinity() == 1:+            return _dec_from_triple(0, '9'*context.prec, context.Etop())++        context = context.copy()+        context._set_rounding(ROUND_FLOOR)+        context._ignore_all_flags()+        new_self = self._fix(context)+        if new_self != self:+            return new_self+        return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1),+                            context)++    def next_plus(self, context=None):+        """Returns the smallest representable number larger than itself."""+        if context is None:+            context = getcontext()++        ans = self._check_nans(context=context)+        if ans:+            return ans++        if self._isinfinity() == 1:+            return _Infinity+        if self._isinfinity() == -1:+            return _dec_from_triple(1, '9'*context.prec, context.Etop())++        context = context.copy()+        context._set_rounding(ROUND_CEILING)+        context._ignore_all_flags()+        new_self = self._fix(context)+        if new_self != self:+            return new_self+        return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1),+                            context)++    def next_toward(self, other, context=None):+        """Returns the number closest to self, in the direction towards other.++        The result is the closest representable number to self+        (excluding self) that is in the direction towards other,+        unless both have the same value.  If the two operands are+        numerically equal, then the result is a copy of self with the+        sign set to be the same as the sign of other.+        """+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return ans++        comparison = self._cmp(other)+        if comparison == 0:+            return self.copy_sign(other)++        if comparison == -1:+            ans = self.next_plus(context)+        else: # comparison == 1+            ans = self.next_minus(context)++        # decide which flags to raise using value of ans+        if ans._isinfinity():+            context._raise_error(Overflow,+                                 'Infinite result from next_toward',+                                 ans._sign)+            context._raise_error(Inexact)+            context._raise_error(Rounded)+        elif ans.adjusted() < context.Emin:+            context._raise_error(Underflow)+            context._raise_error(Subnormal)+            context._raise_error(Inexact)+            context._raise_error(Rounded)+            # if precision == 1 then we don't raise Clamped for a+            # result 0E-Etiny.+            if not ans:+                context._raise_error(Clamped)++        return ans++    def number_class(self, context=None):+        """Returns an indication of the class of self.++        The class is one of the following strings:+          sNaN+          NaN+          -Infinity+          -Normal+          -Subnormal+          -Zero+          +Zero+          +Subnormal+          +Normal+          +Infinity+        """+        if self.is_snan():+            return "sNaN"+        if self.is_qnan():+            return "NaN"+        inf = self._isinfinity()+        if inf == 1:+            return "+Infinity"+        if inf == -1:+            return "-Infinity"+        if self.is_zero():+            if self._sign:+                return "-Zero"+            else:+                return "+Zero"+        if context is None:+            context = getcontext()+        if self.is_subnormal(context=context):+            if self._sign:+                return "-Subnormal"+            else:+                return "+Subnormal"+        # just a normal, regular, boring number, :)+        if self._sign:+            return "-Normal"+        else:+            return "+Normal"++    def radix(self):+        """Just returns 10, as this is Decimal, :)"""+        return Decimal(10)++    def rotate(self, other, context=None):+        """Returns a rotated copy of self, value-of-other times."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if other._exp != 0:+            return context._raise_error(InvalidOperation)+        if not (-context.prec <= int(other) <= context.prec):+            return context._raise_error(InvalidOperation)++        if self._isinfinity():+            return Decimal(self)++        # get values, pad if necessary+        torot = int(other)+        rotdig = self._int+        topad = context.prec - len(rotdig)+        if topad > 0:+            rotdig = '0'*topad + rotdig+        elif topad < 0:+            rotdig = rotdig[-topad:]++        # let's rotate!+        rotated = rotdig[torot:] + rotdig[:torot]+        return _dec_from_triple(self._sign,+                                rotated.lstrip('0') or '0', self._exp)++    def scaleb(self, other, context=None):+        """Returns self operand after adding the second value to its exp."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if other._exp != 0:+            return context._raise_error(InvalidOperation)+        liminf = -2 * (context.Emax + context.prec)+        limsup =  2 * (context.Emax + context.prec)+        if not (liminf <= int(other) <= limsup):+            return context._raise_error(InvalidOperation)++        if self._isinfinity():+            return Decimal(self)++        d = _dec_from_triple(self._sign, self._int, self._exp + int(other))+        d = d._fix(context)+        return d++    def shift(self, other, context=None):+        """Returns a shifted copy of self, value-of-other times."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if other._exp != 0:+            return context._raise_error(InvalidOperation)+        if not (-context.prec <= int(other) <= context.prec):+            return context._raise_error(InvalidOperation)++        if self._isinfinity():+            return Decimal(self)++        # get values, pad if necessary+        torot = int(other)+        rotdig = self._int+        topad = context.prec - len(rotdig)+        if topad > 0:+            rotdig = '0'*topad + rotdig+        elif topad < 0:+            rotdig = rotdig[-topad:]++        # let's shift!+        if torot < 0:+            shifted = rotdig[:torot]+        else:+            shifted = rotdig + '0'*torot+            shifted = shifted[-context.prec:]++        return _dec_from_triple(self._sign,+                                    shifted.lstrip('0') or '0', self._exp)++    # Support for pickling, copy, and deepcopy+    def __reduce__(self):+        return (self.__class__, (str(self),))++    def __copy__(self):+        if type(self) is Decimal:+            return self     # I'm immutable; therefore I am my own clone+        return self.__class__(str(self))++    def __deepcopy__(self, memo):+        if type(self) is Decimal:+            return self     # My components are also immutable+        return self.__class__(str(self))++    # PEP 3101 support.  the _localeconv keyword argument should be+    # considered private: it's provided for ease of testing only.+    def __format__(self, specifier, context=None, _localeconv=None):+        """Format a Decimal instance according to the given specifier.++        The specifier should be a standard format specifier, with the+        form described in PEP 3101.  Formatting types 'e', 'E', 'f',+        'F', 'g', 'G', 'n' and '%' are supported.  If the formatting+        type is omitted it defaults to 'g' or 'G', depending on the+        value of context.capitals.+        """++        # Note: PEP 3101 says that if the type is not present then+        # there should be at least one digit after the decimal point.+        # We take the liberty of ignoring this requirement for+        # Decimal---it's presumably there to make sure that+        # format(float, '') behaves similarly to str(float).+        if context is None:+            context = getcontext()++        spec = _parse_format_specifier(specifier, _localeconv=_localeconv)++        # special values don't care about the type or precision+        if self._is_special:+            sign = _format_sign(self._sign, spec)+            body = str(self.copy_abs())+            if spec['type'] == '%':+                body += '%'+            return _format_align(sign, body, spec)++        # a type of None defaults to 'g' or 'G', depending on context+        if spec['type'] is None:+            spec['type'] = ['g', 'G'][context.capitals]++        # if type is '%', adjust exponent of self accordingly+        if spec['type'] == '%':+            self = _dec_from_triple(self._sign, self._int, self._exp+2)++        # round if necessary, taking rounding mode from the context+        rounding = context.rounding+        precision = spec['precision']+        if precision is not None:+            if spec['type'] in 'eE':+                self = self._round(precision+1, rounding)+            elif spec['type'] in 'fF%':+                self = self._rescale(-precision, rounding)+            elif spec['type'] in 'gG' and len(self._int) > precision:+                self = self._round(precision, rounding)+        # special case: zeros with a positive exponent can't be+        # represented in fixed point; rescale them to 0e0.+        if not self and self._exp > 0 and spec['type'] in 'fF%':+            self = self._rescale(0, rounding)++        # figure out placement of the decimal point+        leftdigits = self._exp + len(self._int)+        if spec['type'] in 'eE':+            if not self and precision is not None:+                dotplace = 1 - precision+            else:+                dotplace = 1+        elif spec['type'] in 'fF%':+            dotplace = leftdigits+        elif spec['type'] in 'gG':+            if self._exp <= 0 and leftdigits > -6:+                dotplace = leftdigits+            else:+                dotplace = 1++        # find digits before and after decimal point, and get exponent+        if dotplace < 0:+            intpart = '0'+            fracpart = '0'*(-dotplace) + self._int+        elif dotplace > len(self._int):+            intpart = self._int + '0'*(dotplace-len(self._int))+            fracpart = ''+        else:+            intpart = self._int[:dotplace] or '0'+            fracpart = self._int[dotplace:]+        exp = leftdigits-dotplace++        # done with the decimal-specific stuff;  hand over the rest+        # of the formatting to the _format_number function+        return _format_number(self._sign, intpart, fracpart, exp, spec)++def _dec_from_triple(sign, coefficient, exponent, special=False):+    """Create a decimal instance directly, without any validation,+    normalization (e.g. removal of leading zeros) or argument+    conversion.++    This function is for *internal use only*.+    """++    self = object.__new__(Decimal)+    self._sign = sign+    self._int = coefficient+    self._exp = exponent+    self._is_special = special++    return self++# Register Decimal as a kind of Number (an abstract base class).+# However, do not register it as Real (because Decimals are not+# interoperable with floats).+_numbers.Number.register(Decimal)+++##### Context class #######################################################++class _ContextManager(object):+    """Context manager class to support localcontext().++      Sets a copy of the supplied context in __enter__() and restores+      the previous decimal context in __exit__()+    """+    def __init__(self, new_context):+        self.new_context = new_context.copy()+    def __enter__(self):+        self.saved_context = getcontext()+        setcontext(self.new_context)+        return self.new_context+    def __exit__(self, t, v, tb):+        setcontext(self.saved_context)++class Context(object):+    """Contains the context for a Decimal instance.++    Contains:+    prec - precision (for use in rounding, division, square roots..)+    rounding - rounding type (how you round)+    traps - If traps[exception] = 1, then the exception is+                    raised when it is caused.  Otherwise, a value is+                    substituted in.+    flags  - When an exception is caused, flags[exception] is set.+             (Whether or not the trap_enabler is set)+             Should be reset by user of Decimal instance.+    Emin -   Minimum exponent+    Emax -   Maximum exponent+    capitals -      If 1, 1*10^1 is printed as 1E+1.+                    If 0, printed as 1e1+    clamp -  If 1, change exponents if too high (Default 0)+    """++    def __init__(self, prec=None, rounding=None, Emin=None, Emax=None,+                       capitals=None, clamp=None, flags=None, traps=None,+                       _ignored_flags=None):+        # Set defaults; for everything except flags and _ignored_flags,+        # inherit from DefaultContext.+        try:+            dc = DefaultContext+        except NameError:+            pass++        self.prec = prec if prec is not None else dc.prec+        self.rounding = rounding if rounding is not None else dc.rounding+        self.Emin = Emin if Emin is not None else dc.Emin+        self.Emax = Emax if Emax is not None else dc.Emax+        self.capitals = capitals if capitals is not None else dc.capitals+        self.clamp = clamp if clamp is not None else dc.clamp++        if _ignored_flags is None:+            self._ignored_flags = []+        else:+            self._ignored_flags = _ignored_flags++        if traps is None:+            self.traps = dc.traps.copy()+        elif not isinstance(traps, dict):+            self.traps = dict((s, int(s in traps)) for s in _signals + traps)+        else:+            self.traps = traps++        if flags is None:+            self.flags = dict.fromkeys(_signals, 0)+        elif not isinstance(flags, dict):+            self.flags = dict((s, int(s in flags)) for s in _signals + flags)+        else:+            self.flags = flags++    def _set_integer_check(self, name, value, vmin, vmax):+        if not isinstance(value, int):+            raise TypeError("%s must be an integer" % name)+        if vmin == '-inf':+            if value > vmax:+                raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value))+        elif vmax == 'inf':+            if value < vmin:+                raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value))+        else:+            if value < vmin or value > vmax:+                raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value))+        return object.__setattr__(self, name, value)++    def _set_signal_dict(self, name, d):+        if not isinstance(d, dict):+            raise TypeError("%s must be a signal dict" % d)+        for key in d:+            if not key in _signals:+                raise KeyError("%s is not a valid signal dict" % d)+        for key in _signals:+            if not key in d:+                raise KeyError("%s is not a valid signal dict" % d)+        return object.__setattr__(self, name, d)++    def __setattr__(self, name, value):+        if name == 'prec':+            return self._set_integer_check(name, value, 1, 'inf')+        elif name == 'Emin':+            return self._set_integer_check(name, value, '-inf', 0)+        elif name == 'Emax':+            return self._set_integer_check(name, value, 0, 'inf')+        elif name == 'capitals':+            return self._set_integer_check(name, value, 0, 1)+        elif name == 'clamp':+            return self._set_integer_check(name, value, 0, 1)+        elif name == 'rounding':+            if not value in _rounding_modes:+                # raise TypeError even for strings to have consistency+                # among various implementations.+                raise TypeError("%s: invalid rounding mode" % value)+            return object.__setattr__(self, name, value)+        elif name == 'flags' or name == 'traps':+            return self._set_signal_dict(name, value)+        elif name == '_ignored_flags':+            return object.__setattr__(self, name, value)+        else:+            raise AttributeError(+                "'decimal.Context' object has no attribute '%s'" % name)++    def __delattr__(self, name):+        raise AttributeError("%s cannot be deleted" % name)++    # Support for pickling, copy, and deepcopy+    def __reduce__(self):+        flags = [sig for sig, v in self.flags.items() if v]+        traps = [sig for sig, v in self.traps.items() if v]+        return (self.__class__,+                (self.prec, self.rounding, self.Emin, self.Emax,+                 self.capitals, self.clamp, flags, traps))++    def __repr__(self):+        """Show the current context."""+        s = []+        s.append('Context(prec=%(prec)d, rounding=%(rounding)s, '+                 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, '+                 'clamp=%(clamp)d'+                 % vars(self))+        names = [f.__name__ for f, v in self.flags.items() if v]+        s.append('flags=[' + ', '.join(names) + ']')+        names = [t.__name__ for t, v in self.traps.items() if v]+        s.append('traps=[' + ', '.join(names) + ']')+        return ', '.join(s) + ')'++    def clear_flags(self):+        """Reset all flags to zero"""+        for flag in self.flags:+            self.flags[flag] = 0++    def clear_traps(self):+        """Reset all traps to zero"""+        for flag in self.traps:+            self.traps[flag] = 0++    def _shallow_copy(self):+        """Returns a shallow copy from self."""+        nc = Context(self.prec, self.rounding, self.Emin, self.Emax,+                     self.capitals, self.clamp, self.flags, self.traps,+                     self._ignored_flags)+        return nc++    def copy(self):+        """Returns a deep copy from self."""+        nc = Context(self.prec, self.rounding, self.Emin, self.Emax,+                     self.capitals, self.clamp,+                     self.flags.copy(), self.traps.copy(),+                     self._ignored_flags)+        return nc+    __copy__ = copy++    def _raise_error(self, condition, explanation = None, *args):+        """Handles an error++        If the flag is in _ignored_flags, returns the default response.+        Otherwise, it sets the flag, then, if the corresponding+        trap_enabler is set, it reraises the exception.  Otherwise, it returns+        the default value after setting the flag.+        """+        error = _condition_map.get(condition, condition)+        if error in self._ignored_flags:+            # Don't touch the flag+            return error().handle(self, *args)++        self.flags[error] = 1+        if not self.traps[error]:+            # The errors define how to handle themselves.+            return condition().handle(self, *args)++        # Errors should only be risked on copies of the context+        # self._ignored_flags = []+        raise error(explanation)++    def _ignore_all_flags(self):+        """Ignore all flags, if they are raised"""+        return self._ignore_flags(*_signals)++    def _ignore_flags(self, *flags):+        """Ignore the flags, if they are raised"""+        # Do not mutate-- This way, copies of a context leave the original+        # alone.+        self._ignored_flags = (self._ignored_flags + list(flags))+        return list(flags)++    def _regard_flags(self, *flags):+        """Stop ignoring the flags, if they are raised"""+        if flags and isinstance(flags[0], (tuple,list)):+            flags = flags[0]+        for flag in flags:+            self._ignored_flags.remove(flag)++    # We inherit object.__hash__, so we must deny this explicitly+    __hash__ = None++    def Etiny(self):+        """Returns Etiny (= Emin - prec + 1)"""+        return int(self.Emin - self.prec + 1)++    def Etop(self):+        """Returns maximum exponent (= Emax - prec + 1)"""+        return int(self.Emax - self.prec + 1)++    def _set_rounding(self, type):+        """Sets the rounding type.++        Sets the rounding type, and returns the current (previous)+        rounding type.  Often used like:++        context = context.copy()+        # so you don't change the calling context+        # if an error occurs in the middle.+        rounding = context._set_rounding(ROUND_UP)+        val = self.__sub__(other, context=context)+        context._set_rounding(rounding)++        This will make it round up for that operation.+        """+        rounding = self.rounding+        self.rounding= type+        return rounding++    def create_decimal(self, num='0'):+        """Creates a new Decimal instance but using self as context.++        This method implements the to-number operation of the+        IBM Decimal specification."""++        if isinstance(num, str) and num != num.strip():+            return self._raise_error(ConversionSyntax,+                                     "no trailing or leading whitespace is "+                                     "permitted.")++        d = Decimal(num, context=self)+        if d._isnan() and len(d._int) > self.prec - self.clamp:+            return self._raise_error(ConversionSyntax,+                                     "diagnostic info too long in NaN")+        return d._fix(self)++    def create_decimal_from_float(self, f):+        """Creates a new Decimal instance from a float but rounding using self+        as the context.++        >>> context = Context(prec=5, rounding=ROUND_DOWN)+        >>> context.create_decimal_from_float(3.1415926535897932)+        Decimal('3.1415')+        >>> context = Context(prec=5, traps=[Inexact])+        >>> context.create_decimal_from_float(3.1415926535897932)+        Traceback (most recent call last):+            ...+        decimal.Inexact: None++        """+        d = Decimal.from_float(f)       # An exact conversion+        return d._fix(self)             # Apply the context rounding++    # Methods+    def abs(self, a):+        """Returns the absolute value of the operand.++        If the operand is negative, the result is the same as using the minus+        operation on the operand.  Otherwise, the result is the same as using+        the plus operation on the operand.++        >>> ExtendedContext.abs(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.abs(Decimal('-100'))+        Decimal('100')+        >>> ExtendedContext.abs(Decimal('101.5'))+        Decimal('101.5')+        >>> ExtendedContext.abs(Decimal('-101.5'))+        Decimal('101.5')+        >>> ExtendedContext.abs(-1)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.__abs__(context=self)++    def add(self, a, b):+        """Return the sum of the two operands.++        >>> ExtendedContext.add(Decimal('12'), Decimal('7.00'))+        Decimal('19.00')+        >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4'))+        Decimal('1.02E+4')+        >>> ExtendedContext.add(1, Decimal(2))+        Decimal('3')+        >>> ExtendedContext.add(Decimal(8), 5)+        Decimal('13')+        >>> ExtendedContext.add(5, 5)+        Decimal('10')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__add__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def _apply(self, a):+        return str(a._fix(self))++    def canonical(self, a):+        """Returns the same Decimal object.++        As we do not have different encodings for the same number, the+        received object already is in its canonical form.++        >>> ExtendedContext.canonical(Decimal('2.50'))+        Decimal('2.50')+        """+        if not isinstance(a, Decimal):+            raise TypeError("canonical requires a Decimal as an argument.")+        return a.canonical()++    def compare(self, a, b):+        """Compares values numerically.++        If the signs of the operands differ, a value representing each operand+        ('-1' if the operand is less than zero, '0' if the operand is zero or+        negative zero, or '1' if the operand is greater than zero) is used in+        place of that operand for the comparison instead of the actual+        operand.++        The comparison is then effected by subtracting the second operand from+        the first and then returning a value according to the result of the+        subtraction: '-1' if the result is less than zero, '0' if the result is+        zero or negative zero, or '1' if the result is greater than zero.++        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3'))+        Decimal('-1')+        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1'))+        Decimal('0')+        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10'))+        Decimal('0')+        >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1'))+        Decimal('1')+        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3'))+        Decimal('1')+        >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1'))+        Decimal('-1')+        >>> ExtendedContext.compare(1, 2)+        Decimal('-1')+        >>> ExtendedContext.compare(Decimal(1), 2)+        Decimal('-1')+        >>> ExtendedContext.compare(1, Decimal(2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.compare(b, context=self)++    def compare_signal(self, a, b):+        """Compares the values of the two operands numerically.++        It's pretty much like compare(), but all NaNs signal, with signaling+        NaNs taking precedence over quiet NaNs.++        >>> c = ExtendedContext+        >>> c.compare_signal(Decimal('2.1'), Decimal('3'))+        Decimal('-1')+        >>> c.compare_signal(Decimal('2.1'), Decimal('2.1'))+        Decimal('0')+        >>> c.flags[InvalidOperation] = 0+        >>> print(c.flags[InvalidOperation])+        0+        >>> c.compare_signal(Decimal('NaN'), Decimal('2.1'))+        Decimal('NaN')+        >>> print(c.flags[InvalidOperation])+        1+        >>> c.flags[InvalidOperation] = 0+        >>> print(c.flags[InvalidOperation])+        0+        >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1'))+        Decimal('NaN')+        >>> print(c.flags[InvalidOperation])+        1+        >>> c.compare_signal(-1, 2)+        Decimal('-1')+        >>> c.compare_signal(Decimal(-1), 2)+        Decimal('-1')+        >>> c.compare_signal(-1, Decimal(2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.compare_signal(b, context=self)++    def compare_total(self, a, b):+        """Compares two operands using their abstract representation.++        This is not like the standard compare, which use their numerical+        value. Note that a total ordering is defined for all possible abstract+        representations.++        >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal('-127'),  Decimal('12'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30'))+        Decimal('0')+        >>> ExtendedContext.compare_total(Decimal('12.3'),  Decimal('12.300'))+        Decimal('1')+        >>> ExtendedContext.compare_total(Decimal('12.3'),  Decimal('NaN'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(1, 2)+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal(1), 2)+        Decimal('-1')+        >>> ExtendedContext.compare_total(1, Decimal(2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.compare_total(b)++    def compare_total_mag(self, a, b):+        """Compares two operands using their abstract representation ignoring sign.++        Like compare_total, but with operand's sign ignored and assumed to be 0.+        """+        a = _convert_other(a, raiseit=True)+        return a.compare_total_mag(b)++    def copy_abs(self, a):+        """Returns a copy of the operand with the sign set to 0.++        >>> ExtendedContext.copy_abs(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.copy_abs(Decimal('-100'))+        Decimal('100')+        >>> ExtendedContext.copy_abs(-1)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.copy_abs()++    def copy_decimal(self, a):+        """Returns a copy of the decimal object.++        >>> ExtendedContext.copy_decimal(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.copy_decimal(Decimal('-1.00'))+        Decimal('-1.00')+        >>> ExtendedContext.copy_decimal(1)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return Decimal(a)++    def copy_negate(self, a):+        """Returns a copy of the operand with the sign inverted.++        >>> ExtendedContext.copy_negate(Decimal('101.5'))+        Decimal('-101.5')+        >>> ExtendedContext.copy_negate(Decimal('-101.5'))+        Decimal('101.5')+        >>> ExtendedContext.copy_negate(1)+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.copy_negate()++    def copy_sign(self, a, b):+        """Copies the second operand's sign to the first one.++        In detail, it returns a copy of the first operand with the sign+        equal to the sign of the second operand.++        >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33'))+        Decimal('1.50')+        >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33'))+        Decimal('1.50')+        >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33'))+        Decimal('-1.50')+        >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33'))+        Decimal('-1.50')+        >>> ExtendedContext.copy_sign(1, -2)+        Decimal('-1')+        >>> ExtendedContext.copy_sign(Decimal(1), -2)+        Decimal('-1')+        >>> ExtendedContext.copy_sign(1, Decimal(-2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.copy_sign(b)++    def divide(self, a, b):+        """Decimal division in a specified context.++        >>> ExtendedContext.divide(Decimal('1'), Decimal('3'))+        Decimal('0.333333333')+        >>> ExtendedContext.divide(Decimal('2'), Decimal('3'))+        Decimal('0.666666667')+        >>> ExtendedContext.divide(Decimal('5'), Decimal('2'))+        Decimal('2.5')+        >>> ExtendedContext.divide(Decimal('1'), Decimal('10'))+        Decimal('0.1')+        >>> ExtendedContext.divide(Decimal('12'), Decimal('12'))+        Decimal('1')+        >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2'))+        Decimal('4.00')+        >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0'))+        Decimal('1.20')+        >>> ExtendedContext.divide(Decimal('1000'), Decimal('100'))+        Decimal('10')+        >>> ExtendedContext.divide(Decimal('1000'), Decimal('1'))+        Decimal('1000')+        >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2'))+        Decimal('1.20E+6')+        >>> ExtendedContext.divide(5, 5)+        Decimal('1')+        >>> ExtendedContext.divide(Decimal(5), 5)+        Decimal('1')+        >>> ExtendedContext.divide(5, Decimal(5))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__truediv__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def divide_int(self, a, b):+        """Divides two numbers and returns the integer part of the result.++        >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3'))+        Decimal('0')+        >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3'))+        Decimal('3')+        >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3'))+        Decimal('3')+        >>> ExtendedContext.divide_int(10, 3)+        Decimal('3')+        >>> ExtendedContext.divide_int(Decimal(10), 3)+        Decimal('3')+        >>> ExtendedContext.divide_int(10, Decimal(3))+        Decimal('3')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__floordiv__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def divmod(self, a, b):+        """Return (a // b, a % b).++        >>> ExtendedContext.divmod(Decimal(8), Decimal(3))+        (Decimal('2'), Decimal('2'))+        >>> ExtendedContext.divmod(Decimal(8), Decimal(4))+        (Decimal('2'), Decimal('0'))+        >>> ExtendedContext.divmod(8, 4)+        (Decimal('2'), Decimal('0'))+        >>> ExtendedContext.divmod(Decimal(8), 4)+        (Decimal('2'), Decimal('0'))+        >>> ExtendedContext.divmod(8, Decimal(4))+        (Decimal('2'), Decimal('0'))+        """+        a = _convert_other(a, raiseit=True)+        r = a.__divmod__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def exp(self, a):+        """Returns e ** a.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.exp(Decimal('-Infinity'))+        Decimal('0')+        >>> c.exp(Decimal('-1'))+        Decimal('0.367879441')+        >>> c.exp(Decimal('0'))+        Decimal('1')+        >>> c.exp(Decimal('1'))+        Decimal('2.71828183')+        >>> c.exp(Decimal('0.693147181'))+        Decimal('2.00000000')+        >>> c.exp(Decimal('+Infinity'))+        Decimal('Infinity')+        >>> c.exp(10)+        Decimal('22026.4658')+        """+        a =_convert_other(a, raiseit=True)+        return a.exp(context=self)++    def fma(self, a, b, c):+        """Returns a multiplied by b, plus c.++        The first two operands are multiplied together, using multiply,+        the third operand is then added to the result of that+        multiplication, using add, all with only one final rounding.++        >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7'))+        Decimal('22')+        >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7'))+        Decimal('-8')+        >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578'))+        Decimal('1.38435736E+12')+        >>> ExtendedContext.fma(1, 3, 4)+        Decimal('7')+        >>> ExtendedContext.fma(1, Decimal(3), 4)+        Decimal('7')+        >>> ExtendedContext.fma(1, 3, Decimal(4))+        Decimal('7')+        """+        a = _convert_other(a, raiseit=True)+        return a.fma(b, c, context=self)++    def is_canonical(self, a):+        """Return True if the operand is canonical; otherwise return False.++        Currently, the encoding of a Decimal instance is always+        canonical, so this method returns True for any Decimal.++        >>> ExtendedContext.is_canonical(Decimal('2.50'))+        True+        """+        if not isinstance(a, Decimal):+            raise TypeError("is_canonical requires a Decimal as an argument.")+        return a.is_canonical()++    def is_finite(self, a):+        """Return True if the operand is finite; otherwise return False.++        A Decimal instance is considered finite if it is neither+        infinite nor a NaN.++        >>> ExtendedContext.is_finite(Decimal('2.50'))+        True+        >>> ExtendedContext.is_finite(Decimal('-0.3'))+        True+        >>> ExtendedContext.is_finite(Decimal('0'))+        True+        >>> ExtendedContext.is_finite(Decimal('Inf'))+        False+        >>> ExtendedContext.is_finite(Decimal('NaN'))+        False+        >>> ExtendedContext.is_finite(1)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_finite()++    def is_infinite(self, a):+        """Return True if the operand is infinite; otherwise return False.++        >>> ExtendedContext.is_infinite(Decimal('2.50'))+        False+        >>> ExtendedContext.is_infinite(Decimal('-Inf'))+        True+        >>> ExtendedContext.is_infinite(Decimal('NaN'))+        False+        >>> ExtendedContext.is_infinite(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_infinite()++    def is_nan(self, a):+        """Return True if the operand is a qNaN or sNaN;+        otherwise return False.++        >>> ExtendedContext.is_nan(Decimal('2.50'))+        False+        >>> ExtendedContext.is_nan(Decimal('NaN'))+        True+        >>> ExtendedContext.is_nan(Decimal('-sNaN'))+        True+        >>> ExtendedContext.is_nan(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_nan()++    def is_normal(self, a):+        """Return True if the operand is a normal number;+        otherwise return False.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.is_normal(Decimal('2.50'))+        True+        >>> c.is_normal(Decimal('0.1E-999'))+        False+        >>> c.is_normal(Decimal('0.00'))+        False+        >>> c.is_normal(Decimal('-Inf'))+        False+        >>> c.is_normal(Decimal('NaN'))+        False+        >>> c.is_normal(1)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_normal(context=self)++    def is_qnan(self, a):+        """Return True if the operand is a quiet NaN; otherwise return False.++        >>> ExtendedContext.is_qnan(Decimal('2.50'))+        False+        >>> ExtendedContext.is_qnan(Decimal('NaN'))+        True+        >>> ExtendedContext.is_qnan(Decimal('sNaN'))+        False+        >>> ExtendedContext.is_qnan(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_qnan()++    def is_signed(self, a):+        """Return True if the operand is negative; otherwise return False.++        >>> ExtendedContext.is_signed(Decimal('2.50'))+        False+        >>> ExtendedContext.is_signed(Decimal('-12'))+        True+        >>> ExtendedContext.is_signed(Decimal('-0'))+        True+        >>> ExtendedContext.is_signed(8)+        False+        >>> ExtendedContext.is_signed(-8)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_signed()++    def is_snan(self, a):+        """Return True if the operand is a signaling NaN;+        otherwise return False.++        >>> ExtendedContext.is_snan(Decimal('2.50'))+        False+        >>> ExtendedContext.is_snan(Decimal('NaN'))+        False+        >>> ExtendedContext.is_snan(Decimal('sNaN'))+        True+        >>> ExtendedContext.is_snan(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_snan()++    def is_subnormal(self, a):+        """Return True if the operand is subnormal; otherwise return False.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.is_subnormal(Decimal('2.50'))+        False+        >>> c.is_subnormal(Decimal('0.1E-999'))+        True+        >>> c.is_subnormal(Decimal('0.00'))+        False+        >>> c.is_subnormal(Decimal('-Inf'))+        False+        >>> c.is_subnormal(Decimal('NaN'))+        False+        >>> c.is_subnormal(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_subnormal(context=self)++    def is_zero(self, a):+        """Return True if the operand is a zero; otherwise return False.++        >>> ExtendedContext.is_zero(Decimal('0'))+        True+        >>> ExtendedContext.is_zero(Decimal('2.50'))+        False+        >>> ExtendedContext.is_zero(Decimal('-0E+2'))+        True+        >>> ExtendedContext.is_zero(1)+        False+        >>> ExtendedContext.is_zero(0)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_zero()++    def ln(self, a):+        """Returns the natural (base e) logarithm of the operand.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.ln(Decimal('0'))+        Decimal('-Infinity')+        >>> c.ln(Decimal('1.000'))+        Decimal('0')+        >>> c.ln(Decimal('2.71828183'))+        Decimal('1.00000000')+        >>> c.ln(Decimal('10'))+        Decimal('2.30258509')+        >>> c.ln(Decimal('+Infinity'))+        Decimal('Infinity')+        >>> c.ln(1)+        Decimal('0')+        """+        a = _convert_other(a, raiseit=True)+        return a.ln(context=self)++    def log10(self, a):+        """Returns the base 10 logarithm of the operand.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.log10(Decimal('0'))+        Decimal('-Infinity')+        >>> c.log10(Decimal('0.001'))+        Decimal('-3')+        >>> c.log10(Decimal('1.000'))+        Decimal('0')+        >>> c.log10(Decimal('2'))+        Decimal('0.301029996')+        >>> c.log10(Decimal('10'))+        Decimal('1')+        >>> c.log10(Decimal('70'))+        Decimal('1.84509804')+        >>> c.log10(Decimal('+Infinity'))+        Decimal('Infinity')+        >>> c.log10(0)+        Decimal('-Infinity')+        >>> c.log10(1)+        Decimal('0')+        """+        a = _convert_other(a, raiseit=True)+        return a.log10(context=self)++    def logb(self, a):+        """ Returns the exponent of the magnitude of the operand's MSD.++        The result is the integer which is the exponent of the magnitude+        of the most significant digit of the operand (as though the+        operand were truncated to a single digit while maintaining the+        value of that digit and without limiting the resulting exponent).++        >>> ExtendedContext.logb(Decimal('250'))+        Decimal('2')+        >>> ExtendedContext.logb(Decimal('2.50'))+        Decimal('0')+        >>> ExtendedContext.logb(Decimal('0.03'))+        Decimal('-2')+        >>> ExtendedContext.logb(Decimal('0'))+        Decimal('-Infinity')+        >>> ExtendedContext.logb(1)+        Decimal('0')+        >>> ExtendedContext.logb(10)+        Decimal('1')+        >>> ExtendedContext.logb(100)+        Decimal('2')+        """+        a = _convert_other(a, raiseit=True)+        return a.logb(context=self)++    def logical_and(self, a, b):+        """Applies the logical operation 'and' between each operand's digits.++        The operands must be both logical numbers.++        >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1'))+        Decimal('0')+        >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010'))+        Decimal('1000')+        >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10'))+        Decimal('10')+        >>> ExtendedContext.logical_and(110, 1101)+        Decimal('100')+        >>> ExtendedContext.logical_and(Decimal(110), 1101)+        Decimal('100')+        >>> ExtendedContext.logical_and(110, Decimal(1101))+        Decimal('100')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_and(b, context=self)++    def logical_invert(self, a):+        """Invert all the digits in the operand.++        The operand must be a logical number.++        >>> ExtendedContext.logical_invert(Decimal('0'))+        Decimal('111111111')+        >>> ExtendedContext.logical_invert(Decimal('1'))+        Decimal('111111110')+        >>> ExtendedContext.logical_invert(Decimal('111111111'))+        Decimal('0')+        >>> ExtendedContext.logical_invert(Decimal('101010101'))+        Decimal('10101010')+        >>> ExtendedContext.logical_invert(1101)+        Decimal('111110010')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_invert(context=self)++    def logical_or(self, a, b):+        """Applies the logical operation 'or' between each operand's digits.++        The operands must be both logical numbers.++        >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0'))+        Decimal('1')+        >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010'))+        Decimal('1110')+        >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10'))+        Decimal('1110')+        >>> ExtendedContext.logical_or(110, 1101)+        Decimal('1111')+        >>> ExtendedContext.logical_or(Decimal(110), 1101)+        Decimal('1111')+        >>> ExtendedContext.logical_or(110, Decimal(1101))+        Decimal('1111')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_or(b, context=self)++    def logical_xor(self, a, b):+        """Applies the logical operation 'xor' between each operand's digits.++        The operands must be both logical numbers.++        >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0'))+        Decimal('1')+        >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1'))+        Decimal('0')+        >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010'))+        Decimal('110')+        >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10'))+        Decimal('1101')+        >>> ExtendedContext.logical_xor(110, 1101)+        Decimal('1011')+        >>> ExtendedContext.logical_xor(Decimal(110), 1101)+        Decimal('1011')+        >>> ExtendedContext.logical_xor(110, Decimal(1101))+        Decimal('1011')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_xor(b, context=self)++    def max(self, a, b):+        """max compares two values numerically and returns the maximum.++        If either operand is a NaN then the general rules apply.+        Otherwise, the operands are compared as though by the compare+        operation.  If they are numerically equal then the left-hand operand+        is chosen as the result.  Otherwise the maximum (closer to positive+        infinity) of the two operands is chosen as the result.++        >>> ExtendedContext.max(Decimal('3'), Decimal('2'))+        Decimal('3')+        >>> ExtendedContext.max(Decimal('-10'), Decimal('3'))+        Decimal('3')+        >>> ExtendedContext.max(Decimal('1.0'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.max(Decimal('7'), Decimal('NaN'))+        Decimal('7')+        >>> ExtendedContext.max(1, 2)+        Decimal('2')+        >>> ExtendedContext.max(Decimal(1), 2)+        Decimal('2')+        >>> ExtendedContext.max(1, Decimal(2))+        Decimal('2')+        """+        a = _convert_other(a, raiseit=True)+        return a.max(b, context=self)++    def max_mag(self, a, b):+        """Compares the values numerically with their sign ignored.++        >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN'))+        Decimal('7')+        >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10'))+        Decimal('-10')+        >>> ExtendedContext.max_mag(1, -2)+        Decimal('-2')+        >>> ExtendedContext.max_mag(Decimal(1), -2)+        Decimal('-2')+        >>> ExtendedContext.max_mag(1, Decimal(-2))+        Decimal('-2')+        """+        a = _convert_other(a, raiseit=True)+        return a.max_mag(b, context=self)++    def min(self, a, b):+        """min compares two values numerically and returns the minimum.++        If either operand is a NaN then the general rules apply.+        Otherwise, the operands are compared as though by the compare+        operation.  If they are numerically equal then the left-hand operand+        is chosen as the result.  Otherwise the minimum (closer to negative+        infinity) of the two operands is chosen as the result.++        >>> ExtendedContext.min(Decimal('3'), Decimal('2'))+        Decimal('2')+        >>> ExtendedContext.min(Decimal('-10'), Decimal('3'))+        Decimal('-10')+        >>> ExtendedContext.min(Decimal('1.0'), Decimal('1'))+        Decimal('1.0')+        >>> ExtendedContext.min(Decimal('7'), Decimal('NaN'))+        Decimal('7')+        >>> ExtendedContext.min(1, 2)+        Decimal('1')+        >>> ExtendedContext.min(Decimal(1), 2)+        Decimal('1')+        >>> ExtendedContext.min(1, Decimal(29))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.min(b, context=self)++    def min_mag(self, a, b):+        """Compares the values numerically with their sign ignored.++        >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2'))+        Decimal('-2')+        >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN'))+        Decimal('-3')+        >>> ExtendedContext.min_mag(1, -2)+        Decimal('1')+        >>> ExtendedContext.min_mag(Decimal(1), -2)+        Decimal('1')+        >>> ExtendedContext.min_mag(1, Decimal(-2))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.min_mag(b, context=self)++    def minus(self, a):+        """Minus corresponds to unary prefix minus in Python.++        The operation is evaluated using the same rules as subtract; the+        operation minus(a) is calculated as subtract('0', a) where the '0'+        has the same exponent as the operand.++        >>> ExtendedContext.minus(Decimal('1.3'))+        Decimal('-1.3')+        >>> ExtendedContext.minus(Decimal('-1.3'))+        Decimal('1.3')+        >>> ExtendedContext.minus(1)+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.__neg__(context=self)++    def multiply(self, a, b):+        """multiply multiplies two operands.++        If either operand is a special value then the general rules apply.+        Otherwise, the operands are multiplied together+        ('long multiplication'), resulting in a number which may be as long as+        the sum of the lengths of the two operands.++        >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3'))+        Decimal('3.60')+        >>> ExtendedContext.multiply(Decimal('7'), Decimal('3'))+        Decimal('21')+        >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8'))+        Decimal('0.72')+        >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0'))+        Decimal('-0.0')+        >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321'))+        Decimal('4.28135971E+11')+        >>> ExtendedContext.multiply(7, 7)+        Decimal('49')+        >>> ExtendedContext.multiply(Decimal(7), 7)+        Decimal('49')+        >>> ExtendedContext.multiply(7, Decimal(7))+        Decimal('49')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__mul__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def next_minus(self, a):+        """Returns the largest representable number smaller than a.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> ExtendedContext.next_minus(Decimal('1'))+        Decimal('0.999999999')+        >>> c.next_minus(Decimal('1E-1007'))+        Decimal('0E-1007')+        >>> ExtendedContext.next_minus(Decimal('-1.00000003'))+        Decimal('-1.00000004')+        >>> c.next_minus(Decimal('Infinity'))+        Decimal('9.99999999E+999')+        >>> c.next_minus(1)+        Decimal('0.999999999')+        """+        a = _convert_other(a, raiseit=True)+        return a.next_minus(context=self)++    def next_plus(self, a):+        """Returns the smallest representable number larger than a.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> ExtendedContext.next_plus(Decimal('1'))+        Decimal('1.00000001')+        >>> c.next_plus(Decimal('-1E-1007'))+        Decimal('-0E-1007')+        >>> ExtendedContext.next_plus(Decimal('-1.00000003'))+        Decimal('-1.00000002')+        >>> c.next_plus(Decimal('-Infinity'))+        Decimal('-9.99999999E+999')+        >>> c.next_plus(1)+        Decimal('1.00000001')+        """+        a = _convert_other(a, raiseit=True)+        return a.next_plus(context=self)++    def next_toward(self, a, b):+        """Returns the number closest to a, in direction towards b.++        The result is the closest representable number from the first+        operand (but not the first operand) that is in the direction+        towards the second operand, unless the operands have the same+        value.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.next_toward(Decimal('1'), Decimal('2'))+        Decimal('1.00000001')+        >>> c.next_toward(Decimal('-1E-1007'), Decimal('1'))+        Decimal('-0E-1007')+        >>> c.next_toward(Decimal('-1.00000003'), Decimal('0'))+        Decimal('-1.00000002')+        >>> c.next_toward(Decimal('1'), Decimal('0'))+        Decimal('0.999999999')+        >>> c.next_toward(Decimal('1E-1007'), Decimal('-100'))+        Decimal('0E-1007')+        >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10'))+        Decimal('-1.00000004')+        >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000'))+        Decimal('-0.00')+        >>> c.next_toward(0, 1)+        Decimal('1E-1007')+        >>> c.next_toward(Decimal(0), 1)+        Decimal('1E-1007')+        >>> c.next_toward(0, Decimal(1))+        Decimal('1E-1007')+        """+        a = _convert_other(a, raiseit=True)+        return a.next_toward(b, context=self)++    def normalize(self, a):+        """normalize reduces an operand to its simplest form.++        Essentially a plus operation with all trailing zeros removed from the+        result.++        >>> ExtendedContext.normalize(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.normalize(Decimal('-2.0'))+        Decimal('-2')+        >>> ExtendedContext.normalize(Decimal('1.200'))+        Decimal('1.2')+        >>> ExtendedContext.normalize(Decimal('-120'))+        Decimal('-1.2E+2')+        >>> ExtendedContext.normalize(Decimal('120.00'))+        Decimal('1.2E+2')+        >>> ExtendedContext.normalize(Decimal('0.00'))+        Decimal('0')+        >>> ExtendedContext.normalize(6)+        Decimal('6')+        """+        a = _convert_other(a, raiseit=True)+        return a.normalize(context=self)++    def number_class(self, a):+        """Returns an indication of the class of the operand.++        The class is one of the following strings:+          -sNaN+          -NaN+          -Infinity+          -Normal+          -Subnormal+          -Zero+          +Zero+          +Subnormal+          +Normal+          +Infinity++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.number_class(Decimal('Infinity'))+        '+Infinity'+        >>> c.number_class(Decimal('1E-10'))+        '+Normal'+        >>> c.number_class(Decimal('2.50'))+        '+Normal'+        >>> c.number_class(Decimal('0.1E-999'))+        '+Subnormal'+        >>> c.number_class(Decimal('0'))+        '+Zero'+        >>> c.number_class(Decimal('-0'))+        '-Zero'+        >>> c.number_class(Decimal('-0.1E-999'))+        '-Subnormal'+        >>> c.number_class(Decimal('-1E-10'))+        '-Normal'+        >>> c.number_class(Decimal('-2.50'))+        '-Normal'+        >>> c.number_class(Decimal('-Infinity'))+        '-Infinity'+        >>> c.number_class(Decimal('NaN'))+        'NaN'+        >>> c.number_class(Decimal('-NaN'))+        'NaN'+        >>> c.number_class(Decimal('sNaN'))+        'sNaN'+        >>> c.number_class(123)+        '+Normal'+        """+        a = _convert_other(a, raiseit=True)+        return a.number_class(context=self)++    def plus(self, a):+        """Plus corresponds to unary prefix plus in Python.++        The operation is evaluated using the same rules as add; the+        operation plus(a) is calculated as add('0', a) where the '0'+        has the same exponent as the operand.++        >>> ExtendedContext.plus(Decimal('1.3'))+        Decimal('1.3')+        >>> ExtendedContext.plus(Decimal('-1.3'))+        Decimal('-1.3')+        >>> ExtendedContext.plus(-1)+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.__pos__(context=self)++    def power(self, a, b, modulo=None):+        """Raises a to the power of b, to modulo if given.++        With two arguments, compute a**b.  If a is negative then b+        must be integral.  The result will be inexact unless b is+        integral and the result is finite and can be expressed exactly+        in 'precision' digits.++        With three arguments, compute (a**b) % modulo.  For the+        three argument form, the following restrictions on the+        arguments hold:++         - all three arguments must be integral+         - b must be nonnegative+         - at least one of a or b must be nonzero+         - modulo must be nonzero and have at most 'precision' digits++        The result of pow(a, b, modulo) is identical to the result+        that would be obtained by computing (a**b) % modulo with+        unbounded precision, but is computed more efficiently.  It is+        always exact.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.power(Decimal('2'), Decimal('3'))+        Decimal('8')+        >>> c.power(Decimal('-2'), Decimal('3'))+        Decimal('-8')+        >>> c.power(Decimal('2'), Decimal('-3'))+        Decimal('0.125')+        >>> c.power(Decimal('1.7'), Decimal('8'))+        Decimal('69.7575744')+        >>> c.power(Decimal('10'), Decimal('0.301029996'))+        Decimal('2.00000000')+        >>> c.power(Decimal('Infinity'), Decimal('-1'))+        Decimal('0')+        >>> c.power(Decimal('Infinity'), Decimal('0'))+        Decimal('1')+        >>> c.power(Decimal('Infinity'), Decimal('1'))+        Decimal('Infinity')+        >>> c.power(Decimal('-Infinity'), Decimal('-1'))+        Decimal('-0')+        >>> c.power(Decimal('-Infinity'), Decimal('0'))+        Decimal('1')+        >>> c.power(Decimal('-Infinity'), Decimal('1'))+        Decimal('-Infinity')+        >>> c.power(Decimal('-Infinity'), Decimal('2'))+        Decimal('Infinity')+        >>> c.power(Decimal('0'), Decimal('0'))+        Decimal('NaN')++        >>> c.power(Decimal('3'), Decimal('7'), Decimal('16'))+        Decimal('11')+        >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16'))+        Decimal('-11')+        >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16'))+        Decimal('1')+        >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16'))+        Decimal('11')+        >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789'))+        Decimal('11729830')+        >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729'))+        Decimal('-0')+        >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537'))+        Decimal('1')+        >>> ExtendedContext.power(7, 7)+        Decimal('823543')+        >>> ExtendedContext.power(Decimal(7), 7)+        Decimal('823543')+        >>> ExtendedContext.power(7, Decimal(7), 2)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__pow__(b, modulo, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def quantize(self, a, b):+        """Returns a value equal to 'a' (rounded), having the exponent of 'b'.++        The coefficient of the result is derived from that of the left-hand+        operand.  It may be rounded using the current rounding setting (if the+        exponent is being increased), multiplied by a positive power of ten (if+        the exponent is being decreased), or is unchanged (if the exponent is+        already equal to that of the right-hand operand).++        Unlike other operations, if the length of the coefficient after the+        quantize operation would be greater than precision then an Invalid+        operation condition is raised.  This guarantees that, unless there is+        an error condition, the exponent of the result of a quantize is always+        equal to that of the right-hand operand.++        Also unlike other operations, quantize will never raise Underflow, even+        if the result is subnormal and inexact.++        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001'))+        Decimal('2.170')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01'))+        Decimal('2.17')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1'))+        Decimal('2.2')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0'))+        Decimal('2')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1'))+        Decimal('0E+1')+        >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity'))+        Decimal('-Infinity')+        >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity'))+        Decimal('NaN')+        >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1'))+        Decimal('-0')+        >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5'))+        Decimal('-0E+5')+        >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2'))+        Decimal('NaN')+        >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2'))+        Decimal('NaN')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1'))+        Decimal('217.0')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0'))+        Decimal('217')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1'))+        Decimal('2.2E+2')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2'))+        Decimal('2E+2')+        >>> ExtendedContext.quantize(1, 2)+        Decimal('1')+        >>> ExtendedContext.quantize(Decimal(1), 2)+        Decimal('1')+        >>> ExtendedContext.quantize(1, Decimal(2))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.quantize(b, context=self)++    def radix(self):+        """Just returns 10, as this is Decimal, :)++        >>> ExtendedContext.radix()+        Decimal('10')+        """+        return Decimal(10)++    def remainder(self, a, b):+        """Returns the remainder from integer division.++        The result is the residue of the dividend after the operation of+        calculating integer division as described for divide-integer, rounded+        to precision digits if necessary.  The sign of the result, if+        non-zero, is the same as that of the original dividend.++        This operation will fail under the same conditions as integer division+        (that is, if integer division on the same two operands would fail, the+        remainder cannot be calculated).++        >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3'))+        Decimal('2.1')+        >>> ExtendedContext.remainder(Decimal('10'), Decimal('3'))+        Decimal('1')+        >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3'))+        Decimal('-1')+        >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1'))+        Decimal('0.2')+        >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3'))+        Decimal('0.1')+        >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3'))+        Decimal('1.0')+        >>> ExtendedContext.remainder(22, 6)+        Decimal('4')+        >>> ExtendedContext.remainder(Decimal(22), 6)+        Decimal('4')+        >>> ExtendedContext.remainder(22, Decimal(6))+        Decimal('4')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__mod__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def remainder_near(self, a, b):+        """Returns to be "a - b * n", where n is the integer nearest the exact+        value of "x / b" (if two integers are equally near then the even one+        is chosen).  If the result is equal to 0 then its sign will be the+        sign of a.++        This operation will fail under the same conditions as integer division+        (that is, if integer division on the same two operands would fail, the+        remainder cannot be calculated).++        >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3'))+        Decimal('-0.9')+        >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6'))+        Decimal('-2')+        >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3'))+        Decimal('1')+        >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3'))+        Decimal('-1')+        >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1'))+        Decimal('0.2')+        >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3'))+        Decimal('0.1')+        >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3'))+        Decimal('-0.3')+        >>> ExtendedContext.remainder_near(3, 11)+        Decimal('3')+        >>> ExtendedContext.remainder_near(Decimal(3), 11)+        Decimal('3')+        >>> ExtendedContext.remainder_near(3, Decimal(11))+        Decimal('3')+        """+        a = _convert_other(a, raiseit=True)+        return a.remainder_near(b, context=self)++    def rotate(self, a, b):+        """Returns a rotated copy of a, b times.++        The coefficient of the result is a rotated copy of the digits in+        the coefficient of the first operand.  The number of places of+        rotation is taken from the absolute value of the second operand,+        with the rotation being to the left if the second operand is+        positive or to the right otherwise.++        >>> ExtendedContext.rotate(Decimal('34'), Decimal('8'))+        Decimal('400000003')+        >>> ExtendedContext.rotate(Decimal('12'), Decimal('9'))+        Decimal('12')+        >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2'))+        Decimal('891234567')+        >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0'))+        Decimal('123456789')+        >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2'))+        Decimal('345678912')+        >>> ExtendedContext.rotate(1333333, 1)+        Decimal('13333330')+        >>> ExtendedContext.rotate(Decimal(1333333), 1)+        Decimal('13333330')+        >>> ExtendedContext.rotate(1333333, Decimal(1))+        Decimal('13333330')+        """+        a = _convert_other(a, raiseit=True)+        return a.rotate(b, context=self)++    def same_quantum(self, a, b):+        """Returns True if the two operands have the same exponent.++        The result is never affected by either the sign or the coefficient of+        either operand.++        >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001'))+        False+        >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01'))+        True+        >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1'))+        False+        >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf'))+        True+        >>> ExtendedContext.same_quantum(10000, -1)+        True+        >>> ExtendedContext.same_quantum(Decimal(10000), -1)+        True+        >>> ExtendedContext.same_quantum(10000, Decimal(-1))+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.same_quantum(b)++    def scaleb (self, a, b):+        """Returns the first operand after adding the second value its exp.++        >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2'))+        Decimal('0.0750')+        >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0'))+        Decimal('7.50')+        >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3'))+        Decimal('7.50E+3')+        >>> ExtendedContext.scaleb(1, 4)+        Decimal('1E+4')+        >>> ExtendedContext.scaleb(Decimal(1), 4)+        Decimal('1E+4')+        >>> ExtendedContext.scaleb(1, Decimal(4))+        Decimal('1E+4')+        """+        a = _convert_other(a, raiseit=True)+        return a.scaleb(b, context=self)++    def shift(self, a, b):+        """Returns a shifted copy of a, b times.++        The coefficient of the result is a shifted copy of the digits+        in the coefficient of the first operand.  The number of places+        to shift is taken from the absolute value of the second operand,+        with the shift being to the left if the second operand is+        positive or to the right otherwise.  Digits shifted into the+        coefficient are zeros.++        >>> ExtendedContext.shift(Decimal('34'), Decimal('8'))+        Decimal('400000000')+        >>> ExtendedContext.shift(Decimal('12'), Decimal('9'))+        Decimal('0')+        >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2'))+        Decimal('1234567')+        >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0'))+        Decimal('123456789')+        >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2'))+        Decimal('345678900')+        >>> ExtendedContext.shift(88888888, 2)+        Decimal('888888800')+        >>> ExtendedContext.shift(Decimal(88888888), 2)+        Decimal('888888800')+        >>> ExtendedContext.shift(88888888, Decimal(2))+        Decimal('888888800')+        """+        a = _convert_other(a, raiseit=True)+        return a.shift(b, context=self)++    def sqrt(self, a):+        """Square root of a non-negative number to context precision.++        If the result must be inexact, it is rounded using the round-half-even+        algorithm.++        >>> ExtendedContext.sqrt(Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.sqrt(Decimal('-0'))+        Decimal('-0')+        >>> ExtendedContext.sqrt(Decimal('0.39'))+        Decimal('0.624499800')+        >>> ExtendedContext.sqrt(Decimal('100'))+        Decimal('10')+        >>> ExtendedContext.sqrt(Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.sqrt(Decimal('1.0'))+        Decimal('1.0')+        >>> ExtendedContext.sqrt(Decimal('1.00'))+        Decimal('1.0')+        >>> ExtendedContext.sqrt(Decimal('7'))+        Decimal('2.64575131')+        >>> ExtendedContext.sqrt(Decimal('10'))+        Decimal('3.16227766')+        >>> ExtendedContext.sqrt(2)+        Decimal('1.41421356')+        >>> ExtendedContext.prec+        9+        """+        a = _convert_other(a, raiseit=True)+        return a.sqrt(context=self)++    def subtract(self, a, b):+        """Return the difference between the two operands.++        >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07'))+        Decimal('0.23')+        >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30'))+        Decimal('0.00')+        >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07'))+        Decimal('-0.77')+        >>> ExtendedContext.subtract(8, 5)+        Decimal('3')+        >>> ExtendedContext.subtract(Decimal(8), 5)+        Decimal('3')+        >>> ExtendedContext.subtract(8, Decimal(5))+        Decimal('3')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__sub__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def to_eng_string(self, a):+        """Convert to a string, using engineering notation if an exponent is needed.++        Engineering notation has an exponent which is a multiple of 3.  This+        can leave up to 3 digits to the left of the decimal place and may+        require the addition of either one or two trailing zeros.++        The operation is not affected by the context.++        >>> ExtendedContext.to_eng_string(Decimal('123E+1'))+        '1.23E+3'+        >>> ExtendedContext.to_eng_string(Decimal('123E+3'))+        '123E+3'+        >>> ExtendedContext.to_eng_string(Decimal('123E-10'))+        '12.3E-9'+        >>> ExtendedContext.to_eng_string(Decimal('-123E-12'))+        '-123E-12'+        >>> ExtendedContext.to_eng_string(Decimal('7E-7'))+        '700E-9'+        >>> ExtendedContext.to_eng_string(Decimal('7E+1'))+        '70'+        >>> ExtendedContext.to_eng_string(Decimal('0E+1'))+        '0.00E+3'++        """+        a = _convert_other(a, raiseit=True)+        return a.to_eng_string(context=self)++    def to_sci_string(self, a):+        """Converts a number to a string, using scientific notation.++        The operation is not affected by the context.+        """+        a = _convert_other(a, raiseit=True)+        return a.__str__(context=self)++    def to_integral_exact(self, a):+        """Rounds to an integer.++        When the operand has a negative exponent, the result is the same+        as using the quantize() operation using the given operand as the+        left-hand-operand, 1E+0 as the right-hand-operand, and the precision+        of the operand as the precision setting; Inexact and Rounded flags+        are allowed in this operation.  The rounding mode is taken from the+        context.++        >>> ExtendedContext.to_integral_exact(Decimal('2.1'))+        Decimal('2')+        >>> ExtendedContext.to_integral_exact(Decimal('100'))+        Decimal('100')+        >>> ExtendedContext.to_integral_exact(Decimal('100.0'))+        Decimal('100')+        >>> ExtendedContext.to_integral_exact(Decimal('101.5'))+        Decimal('102')+        >>> ExtendedContext.to_integral_exact(Decimal('-101.5'))+        Decimal('-102')+        >>> ExtendedContext.to_integral_exact(Decimal('10E+5'))+        Decimal('1.0E+6')+        >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77'))+        Decimal('7.89E+77')+        >>> ExtendedContext.to_integral_exact(Decimal('-Inf'))+        Decimal('-Infinity')+        """+        a = _convert_other(a, raiseit=True)+        return a.to_integral_exact(context=self)++    def to_integral_value(self, a):+        """Rounds to an integer.++        When the operand has a negative exponent, the result is the same+        as using the quantize() operation using the given operand as the+        left-hand-operand, 1E+0 as the right-hand-operand, and the precision+        of the operand as the precision setting, except that no flags will+        be set.  The rounding mode is taken from the context.++        >>> ExtendedContext.to_integral_value(Decimal('2.1'))+        Decimal('2')+        >>> ExtendedContext.to_integral_value(Decimal('100'))+        Decimal('100')+        >>> ExtendedContext.to_integral_value(Decimal('100.0'))+        Decimal('100')+        >>> ExtendedContext.to_integral_value(Decimal('101.5'))+        Decimal('102')+        >>> ExtendedContext.to_integral_value(Decimal('-101.5'))+        Decimal('-102')+        >>> ExtendedContext.to_integral_value(Decimal('10E+5'))+        Decimal('1.0E+6')+        >>> ExtendedContext.to_integral_value(Decimal('7.89E+77'))+        Decimal('7.89E+77')+        >>> ExtendedContext.to_integral_value(Decimal('-Inf'))+        Decimal('-Infinity')+        """+        a = _convert_other(a, raiseit=True)+        return a.to_integral_value(context=self)++    # the method name changed, but we provide also the old one, for compatibility+    to_integral = to_integral_value++class _WorkRep(object):+    __slots__ = ('sign','int','exp')+    # sign: 0 or 1+    # int:  int+    # exp:  None, int, or string++    def __init__(self, value=None):+        if value is None:+            self.sign = None+            self.int = 0+            self.exp = None+        elif isinstance(value, Decimal):+            self.sign = value._sign+            self.int = int(value._int)+            self.exp = value._exp+        else:+            # assert isinstance(value, tuple)+            self.sign = value[0]+            self.int = value[1]+            self.exp = value[2]++    def __repr__(self):+        return "(%r, %r, %r)" % (self.sign, self.int, self.exp)++    __str__ = __repr__++++def _normalize(op1, op2, prec = 0):+    """Normalizes op1, op2 to have the same exp and length of coefficient.++    Done during addition.+    """+    if op1.exp < op2.exp:+        tmp = op2+        other = op1+    else:+        tmp = op1+        other = op2++    # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1).+    # Then adding 10**exp to tmp has the same effect (after rounding)+    # as adding any positive quantity smaller than 10**exp; similarly+    # for subtraction.  So if other is smaller than 10**exp we replace+    # it with 10**exp.  This avoids tmp.exp - other.exp getting too large.+    tmp_len = len(str(tmp.int))+    other_len = len(str(other.int))+    exp = tmp.exp + min(-1, tmp_len - prec - 2)+    if other_len + other.exp - 1 < exp:+        other.int = 1+        other.exp = exp++    tmp.int *= 10 ** (tmp.exp - other.exp)+    tmp.exp = other.exp+    return op1, op2++##### Integer arithmetic functions used by ln, log10, exp and __pow__ #####++_nbits = int.bit_length++def _decimal_lshift_exact(n, e):+    """ Given integers n and e, return n * 10**e if it's an integer, else None.++    The computation is designed to avoid computing large powers of 10+    unnecessarily.++    >>> _decimal_lshift_exact(3, 4)+    30000+    >>> _decimal_lshift_exact(300, -999999999)  # returns None++    """+    if n == 0:+        return 0+    elif e >= 0:+        return n * 10**e+    else:+        # val_n = largest power of 10 dividing n.+        str_n = str(abs(n))+        val_n = len(str_n) - len(str_n.rstrip('0'))+        return None if val_n < -e else n // 10**-e++def _sqrt_nearest(n, a):+    """Closest integer to the square root of the positive integer n.  a is+    an initial approximation to the square root.  Any positive integer+    will do for a, but the closer a is to the square root of n the+    faster convergence will be.++    """+    if n <= 0 or a <= 0:+        raise ValueError("Both arguments to _sqrt_nearest should be positive.")++    b=0+    while a != b:+        b, a = a, a--n//a>>1+    return a++def _rshift_nearest(x, shift):+    """Given an integer x and a nonnegative integer shift, return closest+    integer to x / 2**shift; use round-to-even in case of a tie.++    """+    b, q = 1 << shift, x >> shift+    return q + (2*(x & (b-1)) + (q&1) > b)++def _div_nearest(a, b):+    """Closest integer to a/b, a and b positive integers; rounds to even+    in the case of a tie.++    """+    q, r = divmod(a, b)+    return q + (2*r + (q&1) > b)++def _ilog(x, M, L = 8):+    """Integer approximation to M*log(x/M), with absolute error boundable+    in terms only of x/M.++    Given positive integers x and M, return an integer approximation to+    M * log(x/M).  For L = 8 and 0.1 <= x/M <= 10 the difference+    between the approximation and the exact result is at most 22.  For+    L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15.  In+    both cases these are upper bounds on the error; it will usually be+    much smaller."""++    # The basic algorithm is the following: let log1p be the function+    # log1p(x) = log(1+x).  Then log(x/M) = log1p((x-M)/M).  We use+    # the reduction+    #+    #    log1p(y) = 2*log1p(y/(1+sqrt(1+y)))+    #+    # repeatedly until the argument to log1p is small (< 2**-L in+    # absolute value).  For small y we can use the Taylor series+    # expansion+    #+    #    log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T+    #+    # truncating at T such that y**T is small enough.  The whole+    # computation is carried out in a form of fixed-point arithmetic,+    # with a real number z being represented by an integer+    # approximation to z*M.  To avoid loss of precision, the y below+    # is actually an integer approximation to 2**R*y*M, where R is the+    # number of reductions performed so far.++    y = x-M+    # argument reduction; R = number of reductions performed+    R = 0+    while (R <= L and abs(y) << L-R >= M or+           R > L and abs(y) >> R-L >= M):+        y = _div_nearest((M*y) << 1,+                         M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M))+        R += 1++    # Taylor series with T terms+    T = -int(-10*len(str(M))//(3*L))+    yshift = _rshift_nearest(y, R)+    w = _div_nearest(M, T)+    for k in range(T-1, 0, -1):+        w = _div_nearest(M, k) - _div_nearest(yshift*w, M)++    return _div_nearest(w*y, M)++def _dlog10(c, e, p):+    """Given integers c, e and p with c > 0, p >= 0, compute an integer+    approximation to 10**p * log10(c*10**e), with an absolute error of+    at most 1.  Assumes that c*10**e is not exactly 1."""++    # increase precision by 2; compensate for this by dividing+    # final result by 100+    p += 2++    # write c*10**e as d*10**f with either:+    #   f >= 0 and 1 <= d <= 10, or+    #   f <= 0 and 0.1 <= d <= 1.+    # Thus for c*10**e close to 1, f = 0+    l = len(str(c))+    f = e+l - (e+l >= 1)++    if p > 0:+        M = 10**p+        k = e+p-f+        if k >= 0:+            c *= 10**k+        else:+            c = _div_nearest(c, 10**-k)++        log_d = _ilog(c, M) # error < 5 + 22 = 27+        log_10 = _log10_digits(p) # error < 1+        log_d = _div_nearest(log_d*M, log_10)+        log_tenpower = f*M # exact+    else:+        log_d = 0  # error < 2.31+        log_tenpower = _div_nearest(f, 10**-p) # error < 0.5++    return _div_nearest(log_tenpower+log_d, 100)++def _dlog(c, e, p):+    """Given integers c, e and p with c > 0, compute an integer+    approximation to 10**p * log(c*10**e), with an absolute error of+    at most 1.  Assumes that c*10**e is not exactly 1."""++    # Increase precision by 2. The precision increase is compensated+    # for at the end with a division by 100.+    p += 2++    # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10,+    # or f <= 0 and 0.1 <= d <= 1.  Then we can compute 10**p * log(c*10**e)+    # as 10**p * log(d) + 10**p*f * log(10).+    l = len(str(c))+    f = e+l - (e+l >= 1)++    # compute approximation to 10**p*log(d), with error < 27+    if p > 0:+        k = e+p-f+        if k >= 0:+            c *= 10**k+        else:+            c = _div_nearest(c, 10**-k)  # error of <= 0.5 in c++        # _ilog magnifies existing error in c by a factor of at most 10+        log_d = _ilog(c, 10**p) # error < 5 + 22 = 27+    else:+        # p <= 0: just approximate the whole thing by 0; error < 2.31+        log_d = 0++    # compute approximation to f*10**p*log(10), with error < 11.+    if f:+        extra = len(str(abs(f)))-1+        if p + extra >= 0:+            # error in f * _log10_digits(p+extra) < |f| * 1 = |f|+            # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11+            f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra)+        else:+            f_log_ten = 0+    else:+        f_log_ten = 0++    # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1+    return _div_nearest(f_log_ten + log_d, 100)++class _Log10Memoize(object):+    """Class to compute, store, and allow retrieval of, digits of the+    constant log(10) = 2.302585....  This constant is needed by+    Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__."""+    def __init__(self):+        self.digits = "23025850929940456840179914546843642076011014886"++    def getdigits(self, p):+        """Given an integer p >= 0, return floor(10**p)*log(10).++        For example, self.getdigits(3) returns 2302.+        """+        # digits are stored as a string, for quick conversion to+        # integer in the case that we've already computed enough+        # digits; the stored digits should always be correct+        # (truncated, not rounded to nearest).+        if p < 0:+            raise ValueError("p should be nonnegative")++        if p >= len(self.digits):+            # compute p+3, p+6, p+9, ... digits; continue until at+            # least one of the extra digits is nonzero+            extra = 3+            while True:+                # compute p+extra digits, correct to within 1ulp+                M = 10**(p+extra+2)+                digits = str(_div_nearest(_ilog(10*M, M), 100))+                if digits[-extra:] != '0'*extra:+                    break+                extra += 3+            # keep all reliable digits so far; remove trailing zeros+            # and next nonzero digit+            self.digits = digits.rstrip('0')[:-1]+        return int(self.digits[:p+1])++_log10_digits = _Log10Memoize().getdigits++def _iexp(x, M, L=8):+    """Given integers x and M, M > 0, such that x/M is small in absolute+    value, compute an integer approximation to M*exp(x/M).  For 0 <=+    x/M <= 2.4, the absolute error in the result is bounded by 60 (and+    is usually much smaller)."""++    # Algorithm: to compute exp(z) for a real number z, first divide z+    # by a suitable power R of 2 so that |z/2**R| < 2**-L.  Then+    # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor+    # series+    #+    #     expm1(x) = x + x**2/2! + x**3/3! + ...+    #+    # Now use the identity+    #+    #     expm1(2x) = expm1(x)*(expm1(x)+2)+    #+    # R times to compute the sequence expm1(z/2**R),+    # expm1(z/2**(R-1)), ... , exp(z/2), exp(z).++    # Find R such that x/2**R/M <= 2**-L+    R = _nbits((x<<L)//M)++    # Taylor series.  (2**L)**T > M+    T = -int(-10*len(str(M))//(3*L))+    y = _div_nearest(x, T)+    Mshift = M<<R+    for i in range(T-1, 0, -1):+        y = _div_nearest(x*(Mshift + y), Mshift * i)++    # Expansion+    for k in range(R-1, -1, -1):+        Mshift = M<<(k+2)+        y = _div_nearest(y*(y+Mshift), Mshift)++    return M+y++def _dexp(c, e, p):+    """Compute an approximation to exp(c*10**e), with p decimal places of+    precision.++    Returns integers d, f such that:++      10**(p-1) <= d <= 10**p, and+      (d-1)*10**f < exp(c*10**e) < (d+1)*10**f++    In other words, d*10**f is an approximation to exp(c*10**e) with p+    digits of precision, and with an error in d of at most 1.  This is+    almost, but not quite, the same as the error being < 1ulp: when d+    = 10**(p-1) the error could be up to 10 ulp."""++    # we'll call iexp with M = 10**(p+2), giving p+3 digits of precision+    p += 2++    # compute log(10) with extra precision = adjusted exponent of c*10**e+    extra = max(0, e + len(str(c)) - 1)+    q = p + extra++    # compute quotient c*10**e/(log(10)) = c*10**(e+q)/(log(10)*10**q),+    # rounding down+    shift = e+q+    if shift >= 0:+        cshift = c*10**shift+    else:+        cshift = c//10**-shift+    quot, rem = divmod(cshift, _log10_digits(q))++    # reduce remainder back to original precision+    rem = _div_nearest(rem, 10**extra)++    # error in result of _iexp < 120;  error after division < 0.62+    return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3++def _dpower(xc, xe, yc, ye, p):+    """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and+    y = yc*10**ye, compute x**y.  Returns a pair of integers (c, e) such that:++      10**(p-1) <= c <= 10**p, and+      (c-1)*10**e < x**y < (c+1)*10**e++    in other words, c*10**e is an approximation to x**y with p digits+    of precision, and with an error in c of at most 1.  (This is+    almost, but not quite, the same as the error being < 1ulp: when c+    == 10**(p-1) we can only guarantee error < 10ulp.)++    We assume that: x is positive and not equal to 1, and y is nonzero.+    """++    # Find b such that 10**(b-1) <= |y| <= 10**b+    b = len(str(abs(yc))) + ye++    # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point+    lxc = _dlog(xc, xe, p+b+1)++    # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1)+    shift = ye-b+    if shift >= 0:+        pc = lxc*yc*10**shift+    else:+        pc = _div_nearest(lxc*yc, 10**-shift)++    if pc == 0:+        # we prefer a result that isn't exactly 1; this makes it+        # easier to compute a correctly rounded result in __pow__+        if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1:+            coeff, exp = 10**(p-1)+1, 1-p+        else:+            coeff, exp = 10**p-1, -p+    else:+        coeff, exp = _dexp(pc, -(p+1), p+1)+        coeff = _div_nearest(coeff, 10)+        exp += 1++    return coeff, exp++def _log10_lb(c, correction = {+        '1': 100, '2': 70, '3': 53, '4': 40, '5': 31,+        '6': 23, '7': 16, '8': 10, '9': 5}):+    """Compute a lower bound for 100*log10(c) for a positive integer c."""+    if c <= 0:+        raise ValueError("The argument to _log10_lb should be nonnegative.")+    str_c = str(c)+    return 100*len(str_c) - correction[str_c[0]]++##### Helper Functions ####################################################++def _convert_other(other, raiseit=False, allow_float=False):+    """Convert other to Decimal.++    Verifies that it's ok to use in an implicit construction.+    If allow_float is true, allow conversion from float;  this+    is used in the comparison methods (__eq__ and friends).++    """+    if isinstance(other, Decimal):+        return other+    if isinstance(other, int):+        return Decimal(other)+    if allow_float and isinstance(other, float):+        return Decimal.from_float(other)++    if raiseit:+        raise TypeError("Unable to convert %s to Decimal" % other)+    return NotImplemented++def _convert_for_comparison(self, other, equality_op=False):+    """Given a Decimal instance self and a Python object other, return+    a pair (s, o) of Decimal instances such that "s op o" is+    equivalent to "self op other" for any of the 6 comparison+    operators "op".++    """+    if isinstance(other, Decimal):+        return self, other++    # Comparison with a Rational instance (also includes integers):+    # self op n/d <=> self*d op n (for n and d integers, d positive).+    # A NaN or infinity can be left unchanged without affecting the+    # comparison result.+    if isinstance(other, _numbers.Rational):+        if not self._is_special:+            self = _dec_from_triple(self._sign,+                                    str(int(self._int) * other.denominator),+                                    self._exp)+        return self, Decimal(other.numerator)++    # Comparisons with float and complex types.  == and != comparisons+    # with complex numbers should succeed, returning either True or False+    # as appropriate.  Other comparisons return NotImplemented.+    if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0:+        other = other.real+    if isinstance(other, float):+        context = getcontext()+        if equality_op:+            context.flags[FloatOperation] = 1+        else:+            context._raise_error(FloatOperation,+                "strict semantics for mixing floats and Decimals are enabled")+        return self, Decimal.from_float(other)+    return NotImplemented, NotImplemented+++##### Setup Specific Contexts ############################################++# The default context prototype used by Context()+# Is mutable, so that new contexts can have different default values++DefaultContext = Context(+        prec=28, rounding=ROUND_HALF_EVEN,+        traps=[DivisionByZero, Overflow, InvalidOperation],+        flags=[],+        Emax=999999,+        Emin=-999999,+        capitals=1,+        clamp=0+)++# Pre-made alternate contexts offered by the specification+# Don't change these; the user should be able to select these+# contexts and be able to reproduce results from other implementations+# of the spec.++BasicContext = Context(+        prec=9, rounding=ROUND_HALF_UP,+        traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow],+        flags=[],+)++ExtendedContext = Context(+        prec=9, rounding=ROUND_HALF_EVEN,+        traps=[],+        flags=[],+)+++##### crud for parsing strings #############################################+#+# Regular expression used for parsing numeric strings.  Additional+# comments:+#+# 1. Uncomment the two '\s*' lines to allow leading and/or trailing+# whitespace.  But note that the specification disallows whitespace in+# a numeric string.+#+# 2. For finite numbers (not infinities and NaNs) the body of the+# number between the optional sign and the optional exponent must have+# at least one decimal digit, possibly after the decimal point.  The+# lookahead expression '(?=\d|\.\d)' checks this.++import re+_parser = re.compile(r"""        # A numeric string consists of:+#    \s*+    (?P<sign>[-+])?              # an optional sign, followed by either...+    (+        (?=\d|\.\d)              # ...a number (with at least one digit)+        (?P<int>\d*)             # having a (possibly empty) integer part+        (\.(?P<frac>\d*))?       # followed by an optional fractional part+        (E(?P<exp>[-+]?\d+))?    # followed by an optional exponent, or...+    |+        Inf(inity)?              # ...an infinity, or...+    |+        (?P<signal>s)?           # ...an (optionally signaling)+        NaN                      # NaN+        (?P<diag>\d*)            # with (possibly empty) diagnostic info.+    )+#    \s*+    \Z+""", re.VERBOSE | re.IGNORECASE).match++_all_zeros = re.compile('0*$').match+_exact_half = re.compile('50*$').match++##### PEP3101 support functions ##############################################+# The functions in this section have little to do with the Decimal+# class, and could potentially be reused or adapted for other pure+# Python numeric classes that want to implement __format__+#+# A format specifier for Decimal looks like:+#+#   [[fill]align][sign][#][0][minimumwidth][,][.precision][type]++_parse_format_specifier_regex = re.compile(r"""\A+(?:+   (?P<fill>.)?+   (?P<align>[<>=^])+)?+(?P<sign>[-+ ])?+(?P<alt>\#)?+(?P<zeropad>0)?+(?P<minimumwidth>(?!0)\d+)?+(?P<thousands_sep>,)?+(?:\.(?P<precision>0|(?!0)\d+))?+(?P<type>[eEfFgGn%])?+\Z+""", re.VERBOSE|re.DOTALL)++del re++# The locale module is only needed for the 'n' format specifier.  The+# rest of the PEP 3101 code functions quite happily without it, so we+# don't care too much if locale isn't present.+try:+    import locale as _locale+except ImportError:+    pass++def _parse_format_specifier(format_spec, _localeconv=None):+    """Parse and validate a format specifier.++    Turns a standard numeric format specifier into a dict, with the+    following entries:++      fill: fill character to pad field to minimum width+      align: alignment type, either '<', '>', '=' or '^'+      sign: either '+', '-' or ' '+      minimumwidth: nonnegative integer giving minimum width+      zeropad: boolean, indicating whether to pad with zeros+      thousands_sep: string to use as thousands separator, or ''+      grouping: grouping for thousands separators, in format+        used by localeconv+      decimal_point: string to use for decimal point+      precision: nonnegative integer giving precision, or None+      type: one of the characters 'eEfFgG%', or None++    """+    m = _parse_format_specifier_regex.match(format_spec)+    if m is None:+        raise ValueError("Invalid format specifier: " + format_spec)++    # get the dictionary+    format_dict = m.groupdict()++    # zeropad; defaults for fill and alignment.  If zero padding+    # is requested, the fill and align fields should be absent.+    fill = format_dict['fill']+    align = format_dict['align']+    format_dict['zeropad'] = (format_dict['zeropad'] is not None)+    if format_dict['zeropad']:+        if fill is not None:+            raise ValueError("Fill character conflicts with '0'"+                             " in format specifier: " + format_spec)+        if align is not None:+            raise ValueError("Alignment conflicts with '0' in "+                             "format specifier: " + format_spec)+    format_dict['fill'] = fill or ' '+    # PEP 3101 originally specified that the default alignment should+    # be left;  it was later agreed that right-aligned makes more sense+    # for numeric types.  See http://bugs.python.org/issue6857.+    format_dict['align'] = align or '>'++    # default sign handling: '-' for negative, '' for positive+    if format_dict['sign'] is None:+        format_dict['sign'] = '-'++    # minimumwidth defaults to 0; precision remains None if not given+    format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0')+    if format_dict['precision'] is not None:+        format_dict['precision'] = int(format_dict['precision'])++    # if format type is 'g' or 'G' then a precision of 0 makes little+    # sense; convert it to 1.  Same if format type is unspecified.+    if format_dict['precision'] == 0:+        if format_dict['type'] is None or format_dict['type'] in 'gGn':+            format_dict['precision'] = 1++    # determine thousands separator, grouping, and decimal separator, and+    # add appropriate entries to format_dict+    if format_dict['type'] == 'n':+        # apart from separators, 'n' behaves just like 'g'+        format_dict['type'] = 'g'+        if _localeconv is None:+            _localeconv = _locale.localeconv()+        if format_dict['thousands_sep'] is not None:+            raise ValueError("Explicit thousands separator conflicts with "+                             "'n' type in format specifier: " + format_spec)+        format_dict['thousands_sep'] = _localeconv['thousands_sep']+        format_dict['grouping'] = _localeconv['grouping']+        format_dict['decimal_point'] = _localeconv['decimal_point']+    else:+        if format_dict['thousands_sep'] is None:+            format_dict['thousands_sep'] = ''+        format_dict['grouping'] = [3, 0]+        format_dict['decimal_point'] = '.'++    return format_dict++def _format_align(sign, body, spec):+    """Given an unpadded, non-aligned numeric string 'body' and sign+    string 'sign', add padding and alignment conforming to the given+    format specifier dictionary 'spec' (as produced by+    parse_format_specifier).++    """+    # how much extra space do we have to play with?+    minimumwidth = spec['minimumwidth']+    fill = spec['fill']+    padding = fill*(minimumwidth - len(sign) - len(body))++    align = spec['align']+    if align == '<':+        result = sign + body + padding+    elif align == '>':+        result = padding + sign + body+    elif align == '=':+        result = sign + padding + body+    elif align == '^':+        half = len(padding)//2+        result = padding[:half] + sign + body + padding[half:]+    else:+        raise ValueError('Unrecognised alignment field')++    return result++def _group_lengths(grouping):+    """Convert a localeconv-style grouping into a (possibly infinite)+    iterable of integers representing group lengths.++    """+    # The result from localeconv()['grouping'], and the input to this+    # function, should be a list of integers in one of the+    # following three forms:+    #+    #   (1) an empty list, or+    #   (2) nonempty list of positive integers + [0]+    #   (3) list of positive integers + [locale.CHAR_MAX], or++    from itertools import chain, repeat+    if not grouping:+        return []+    elif grouping[-1] == 0 and len(grouping) >= 2:+        return chain(grouping[:-1], repeat(grouping[-2]))+    elif grouping[-1] == _locale.CHAR_MAX:+        return grouping[:-1]+    else:+        raise ValueError('unrecognised format for grouping')++def _insert_thousands_sep(digits, spec, min_width=1):+    """Insert thousands separators into a digit string.++    spec is a dictionary whose keys should include 'thousands_sep' and+    'grouping'; typically it's the result of parsing the format+    specifier using _parse_format_specifier.++    The min_width keyword argument gives the minimum length of the+    result, which will be padded on the left with zeros if necessary.++    If necessary, the zero padding adds an extra '0' on the left to+    avoid a leading thousands separator.  For example, inserting+    commas every three digits in '123456', with min_width=8, gives+    '0,123,456', even though that has length 9.++    """++    sep = spec['thousands_sep']+    grouping = spec['grouping']++    groups = []+    for l in _group_lengths(grouping):+        if l <= 0:+            raise ValueError("group length should be positive")+        # max(..., 1) forces at least 1 digit to the left of a separator+        l = min(max(len(digits), min_width, 1), l)+        groups.append('0'*(l - len(digits)) + digits[-l:])+        digits = digits[:-l]+        min_width -= l+        if not digits and min_width <= 0:+            break+        min_width -= len(sep)+    else:+        l = max(len(digits), min_width, 1)+        groups.append('0'*(l - len(digits)) + digits[-l:])+    return sep.join(reversed(groups))++def _format_sign(is_negative, spec):+    """Determine sign character."""++    if is_negative:+        return '-'+    elif spec['sign'] in ' +':+        return spec['sign']+    else:+        return ''++def _format_number(is_negative, intpart, fracpart, exp, spec):+    """Format a number, given the following data:++    is_negative: true if the number is negative, else false+    intpart: string of digits that must appear before the decimal point+    fracpart: string of digits that must come after the point+    exp: exponent, as an integer+    spec: dictionary resulting from parsing the format specifier++    This function uses the information in spec to:+      insert separators (decimal separator and thousands separators)+      format the sign+      format the exponent+      add trailing '%' for the '%' type+      zero-pad if necessary+      fill and align if necessary+    """++    sign = _format_sign(is_negative, spec)++    if fracpart or spec['alt']:+        fracpart = spec['decimal_point'] + fracpart++    if exp != 0 or spec['type'] in 'eE':+        echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']]+        fracpart += "{0}{1:+}".format(echar, exp)+    if spec['type'] == '%':+        fracpart += '%'++    if spec['zeropad']:+        min_width = spec['minimumwidth'] - len(fracpart) - len(sign)+    else:+        min_width = 0+    intpart = _insert_thousands_sep(intpart, spec, min_width)++    return _format_align(sign, intpart+fracpart, spec)+++##### Useful Constants (internal use only) ################################++# Reusable defaults+_Infinity = Decimal('Inf')+_NegativeInfinity = Decimal('-Inf')+_NaN = Decimal('NaN')+_Zero = Decimal(0)+_One = Decimal(1)+_NegativeOne = Decimal(-1)++# _SignedInfinity[sign] is infinity w/ that sign+_SignedInfinity = (_Infinity, _NegativeInfinity)++# Constants related to the hash implementation;  hash(x) is based+# on the reduction of x modulo _PyHASH_MODULUS+_PyHASH_MODULUS = sys.hash_info.modulus+# hash values to use for positive and negative infinities, and nans+_PyHASH_INF = sys.hash_info.inf+_PyHASH_NAN = sys.hash_info.nan++# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS+_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)+del sys
+ example/FixMutableDefaultArguments.hs view
@@ -0,0 +1,201 @@+{-# language DataKinds #-}+module FixMutableDefaultArguments where++import Control.Lens.Cons (_head)+import Control.Lens.Fold ((^..), (^?), filtered, folded, anyOf, has)+import Control.Lens.Getter ((^.))+import Control.Lens.Review ((#))+import Control.Lens.Setter ((.~), (%~))+import Control.Monad (guard)+import Data.Function ((&))+import Data.Semigroup ((<>))++import Language.Python.DSL+import Language.Python.Optics+import Language.Python.Syntax.Expr (Expr(..), _Exprs)++{-++I want to write a function that fixes the 'mutable default argument' pattern.++It takes a function like this:++def a(b=[]):+  b.push(1)++to this:++def a(b=None):+  if b is None:+    b = []+  b.push(1)++if the parameter is a 'mutable' thing.++-}+fixMutableDefaultArguments :: Raw Statement -> Maybe (Raw Statement)+fixMutableDefaultArguments input = do+  {-++  Firstly, this transformation only applies to function definitions. So we use the '_Fundef'+  prism to pull out the function definition from a statement.++  -}+  function <- input ^? _Fundef++  let++    {-++    I want the parameters of the function as a list. I can use the 'parameters' lens to get+    at them.++    -}+    paramsList = function ^.. parameters.folded++    {-++    I also want to know which parameters meet the 'mutable default argument' pattern.++    The '_KeywordParam' prism only matches keyword parameters, and '_kpExpr' is the field+    accessor for the right hand side of an '=' in a keyword parameter.++    This expression gets 'all the keyword parameters that have mutable value on their RHS'++    -}+    targetParams = paramsList ^.. folded._KeywordParam.filtered (isMutable._kpExpr)++  {-++  If the list of targetParams is empty, then we don't need to do the transformation++  -}+  guard $ has _head targetParams++  let+    {-++    Let's generate some 'if' statements++    -}+    conditionalAssignments =+      {-++      for each 'param' in our list of keyword parameters++      -}+      (\param ->+         let+           {-++           let <pName> be the left hand side of the '='++           -}+           pName = var_ (param ^. kpName.identValue)++           {-++           let <pValue> be the right hand side of the '='++           -}+           pValue = param ^. kpExpr++         in+           {-++           output a new line, which says...++           -}+           line_ $+           {-++           if <pName> is None:+               <pName> = <pValue>++           -}+           if_ (pName `is_` none_) [ line_ (pName .= pValue) ]) <$>+      targetParams++    {-++    For each parameter in the original parameter list, set the right hand sides of+    the target parameters to 'None', but leave all of the others as they were.++    -}+    newparams =+      paramsList & traverse._KeywordParam.kpExpr.filtered isMutable .~ none_++  pure $+    {-++    Return a new function defition statement++    -}+    _Fundef #+      {-++      that consists of the original function++      -}+      (function &+       {-++       with its problem parameters set to 'None'++       -}+       parameters_ .~ newparams &++       {-++       and for each problem parameter, there is a corresponding if statement at the+       start of the function definition++       -}+       body_ %~ (conditionalAssignments <>))++  where+    {-++    This function decides whether or not an expression is mutable++    -}+    isMutable :: Raw Expr -> Bool+    isMutable Unit{} = False+    isMutable None{} = False+    isMutable Ellipsis{} = False+    isMutable Lambda{} = False+    isMutable Float{} = False+    isMutable Imag{} = False+    isMutable Int{} = False+    isMutable Bool{} = False+    isMutable String{} = False++    isMutable List{} = True+    isMutable ListComp{} = True+    isMutable Deref{} = True+    isMutable Call{} = True+    isMutable BinOp{} = True+    isMutable UnOp{} = True+    isMutable Not{} = True+    isMutable DictComp{} = True+    isMutable Dict{} = True+    isMutable Ident{} = True+    isMutable Yield{} = True+    isMutable Await{} = True+    isMutable YieldFrom{} = True+    isMutable SetComp{} = True+    isMutable Set{} = True+    isMutable Subscript{} = True+    isMutable Generator{} = True++    isMutable (Ternary _ _ _ a _ b) = isMutable a || isMutable b+    isMutable (Parens _ _ a _) = isMutable a+    isMutable (Tuple _ a _ as) =+      {-++      Tuples contain many expressions, and are mutable if any of the sub-expressions+      are mutable. The '_Exprs' traversal can get at all these sub-expressions++      -}+      anyOf _Exprs isMutable a ||+      anyOf (folded.folded._Exprs) isMutable as
+ example/Indentation.hs view
@@ -0,0 +1,27 @@+{-# language DataKinds #-}+module Indentation where++import Control.Lens.Setter ((.~))+import Control.Lens.Plated (transform)+import GHC.Natural (Natural)++import Language.Python.Optics+import Language.Python.Syntax.Statement (Statement)+import Language.Python.Syntax.Whitespace (Whitespace (Space, Tab))++{-++These functions show how we can use Control.Lens.Plated to perform+whole-program transformations.++They're illustrative only, because these functions aren't enough to re-indent+all Python code properly. The _Indent optic is limited in where it can reach. See+Language.Python.Optics.Indents for more info.++-}++indentSpaces :: Natural -> Statement '[] a -> Statement '[] a+indentSpaces n = transform (_Indent .~ replicate (fromIntegral n) Space)++indentTabs :: Statement '[] a -> Statement '[] a+indentTabs = transform (_Indent .~ [Tab])
+ example/Main.hs view
@@ -0,0 +1,49 @@+{-# language DataKinds #-}+module Main where++import Control.Lens++import Programs+import FixMutableDefaultArguments+import OptimizeTailRecursion+import Indentation+import Validation++import Language.Python.Render (showModule)+import Language.Python.Syntax.Statement (_Statements)++import qualified Data.Text.IO as StrictText++section a = do+  putStrLn "**********"+  a+  putStrLn "\n**********\n"++main = do+  section $ do+    putStrLn "Before\n"+    StrictText.putStrLn $ showModule everything++  section $ do+    putStrLn "Spaced\n"+    StrictText.putStrLn .+      showModule $+      everything & _Statements %~ indentSpaces 2++  section $ do+    putStrLn "Tabbed\n"+    StrictText.putStrLn .+      showModule $+      everything & _Statements %~ indentTabs++  section $ do+    putStrLn "Refactored\n"+    StrictText.putStrLn .+      showModule .+      rewriteOn _Statements fixMutableDefaultArguments .+      rewriteOn _Statements optimizeTailRecursion $+      everything++  section $ do+    putStrLn "Validated\n"+    doValidating
+ example/OptimizeTailRecursion.hs view
@@ -0,0 +1,149 @@+{-# language OverloadedStrings #-}+{-# language DataKinds #-}+{-# language BangPatterns #-}+module OptimizeTailRecursion where++import Control.Applicative ((<|>))+import Control.Lens.Cons (_last, _init)+import Control.Lens.Fold ((^..), (^?), (^?!), allOf, anyOf, folded, foldrOf)+import Control.Lens.Getter ((^.), to)+import Control.Lens.Plated (cosmos, transform, transformOn)+import Control.Lens.Prism (_Just)+import Control.Lens.Review ((#))+import Control.Lens.Setter ((%~), (.~))+import Control.Lens.Tuple (_2, _3)+import Data.Foldable (toList)+import Data.Function ((&))+import Data.Semigroup ((<>))++import Language.Python.Optics+import Language.Python.DSL+import Language.Python.Syntax.Expr (Expr (..), _Exprs, argExpr, paramName)+import Language.Python.Syntax.Statement (CompoundStatement (..), Statement (..), SmallStatement (..), SimpleStatement (..), _Statements)++optimizeTailRecursion :: Raw Statement -> Maybe (Raw Statement)+optimizeTailRecursion st = do+  function <- st ^? _Fundef+  let functionBody = function ^. body_+  bodyLast <- lastStatement functionBody++  let+    functionName = function ^. fdName.identValue+    bodyInit = functionBody ^?! _init+    paramNames = function ^.. fdParameters.folded.paramName.identValue++  if not $ hasTC functionName bodyLast+    then Nothing+    else+      Just $+      _Fundef #+        (function &+         body_ .~+           (zipWith+              (\a b -> line_ (var_ (a <> "__tr") .= var_ b))+              paramNames+              paramNames <>++            [ line_ ("__res__tr" .= none_)+            , line_ . while_ true_ .+              transformOn (traverse._Exprs) (renameIn paramNames "__tr") $+                bodyInit <>+                looped functionName paramNames bodyLast+            , line_ $ return_ "__res__tr"+            ]))++  where+    lastStatement :: [Raw Line] -> Maybe (Raw Statement)+    lastStatement = go Nothing+      where+        go !res [] = res+        go !res (a:as) = go (a ^? _Statements <|> res) as++    isTailCall :: String -> Raw Expr -> Bool+    isTailCall name e+      | anyOf (cosmos._Call.callFunction._Ident.identValue) (== name) e+      = (e ^? _Call.callFunction._Ident.identValue) == Just name+      | otherwise = False++    hasTC :: String -> Raw Statement -> Bool+    hasTC name st =+      case st of+        CompoundStatement (If _ _ _ _ sts [] sts') ->+          allOf _last (hasTC name) (sts ^.. _Statements) ||+          allOf _last (hasTC name) (sts' ^.. _Just._3._Statements)+        SmallStatement _ (MkSmallStatement s ss _ _ _) ->+          case last (s : fmap (^. _2) ss) of+            Return _ _ (Just e) -> isTailCall name e+            -- Return _ _ Nothing -> True+            Expr _ e -> isTailCall name e+            _ -> False+        _ -> False++    renameIn :: [String] -> String -> Raw Expr -> Raw Expr+    renameIn params suffix =+      transform+        (_Ident.identValue %~ (\a -> if a `elem` params then a <> suffix else a))++    looped :: String -> [String] -> Raw Statement -> [Raw Line]+    looped name params st+      | Just ifSt <- st ^? _If+      , hasTC name st =+          let+            ifBodyLines = toList $ ifSt ^. body_+          in+            case ifSt ^? to getElse._Just.body_ of+              Nothing ->+                [ line_ $+                  if_ (ifSt ^. ifCond)+                    ((ifBodyLines ^?! _init) <>+                     looped name params (ifBodyLines ^?! _last._Statements))+                ]+              Just sts'' ->+                [ line_ $+                  if_ (ifSt ^. ifCond)+                    ((ifSt ^?! body_.to toList._init) <>+                     looped name params (ifBodyLines ^?! _last._Statements)) &+                  else_+                    ((toList sts'' ^?! _init) <>+                     looped name params (toList sts'' ^?! _last._Statements))+                ]+      | otherwise =+          case st of+            CompoundStatement{} -> [line_ st]+            SmallStatement idnts (MkSmallStatement s ss sc cmt nl) ->+              let+                initExps = foldr (\_ _ -> init ss) [] ss+                lastExp = foldrOf (folded._2) (\_ _ -> last ss ^. _2) s ss+                newSts =+                  case initExps of+                    [] -> []+                    first : rest ->+                      [ line_ $+                        SmallStatement idnts+                        (MkSmallStatement (first ^. _2) rest sc cmt nl)+                      ]+              in+                case lastExp of+                  Return _ _ e ->+                    case e ^? _Just._Call of+                      Just call+                        | Just name' <- call ^? callFunction._Ident.identValue+                        , name' == name ->+                            newSts <>+                            fmap+                              (\a -> line_ (var_ (a <> "__tr__old") .= var_ (a <> "__tr")))+                              params <>+                            zipWith+                              (\a b -> line_ (var_ (a <> "__tr") .= b))+                              params+                              (transformOn+                                traverse+                                (renameIn params "__tr__old")+                                (call ^.. callArguments.folded.folded.argExpr))+                      _ ->+                        newSts <>+                        maybe [] (\e' -> [ line_ ("__res__tr" .= e') ]) e <>+                        [ line_ break_ ]+                  Expr _ e+                    | isTailCall name e -> newSts <> [line_ pass_]+                  _ -> [line_ st]
+ example/Programs.hs view
@@ -0,0 +1,208 @@+{-# language OverloadedStrings #-}+{-# language FlexibleContexts #-}+module Programs where++import Control.Lens.Getter ((^.))+import Control.Lens.Iso (from)+import Data.Function ((&))+import Data.List.NonEmpty (NonEmpty(..))++import Language.Python.DSL+import Language.Python.Syntax+++-- |+-- @+-- def append_to(element, to=[]):+--   to.append(element)+--   return to+-- @+--+-- Written without the DSL (not recommended!)+append_to :: Raw Statement+append_to =+  CompoundStatement $+  Fundef () [] (Indents [] ())+    Nothing+    (Space :| [])+    "append_to"+    []+    ( CommaSepMany (PositionalParam () "element" Nothing) (MkComma [Space]) $+      CommaSepOne (KeywordParam () "to" Nothing [] (List () [] Nothing []))+    )+    []+    Nothing+    (SuiteMany () (MkColon []) Nothing LF $+     Block []+     ( SmallStatement+         (Indents [replicate 4 Space ^. from indentWhitespaces] ())+         (MkSmallStatement+          (Expr () $+           Call ()+             (Deref () (Ident "to") [] "append")+             []+             (Just $ CommaSepOne1' (PositionalArg () (Ident "element")) Nothing)+             [])+          []+          Nothing+          Nothing+          (Just LF))+     )+     [ Right $+         SmallStatement+           (Indents [replicate 4 Space ^. from indentWhitespaces] ())+           (MkSmallStatement+            (Return () [Space] (Just $ Ident "to"))+            []+            Nothing+            Nothing+            (Just LF))+     ])++-- |+-- @+-- def append_to(element, to=[]):+--   to.append(element)+--   return to+-- @+--+-- Written with the DSL+append_to' :: Raw Fundef+append_to' =+  def_ "append_to" [ p_ "element", k_ "to" (list_ []) ]+    [ line_ $ call_ ("to" /> "append") [ "element" ]+    , line_ $ return_ "to"+    ]++-- |+-- @+-- def fact(n)+--   def go(n, acc)+--     if n == 0:+--       return acc+--     else:+--       go(n-1, n*acc)+--   return go(n, 1)+-- @+fact_tr :: Raw Fundef+fact_tr =+  def_ "fact" [p_ "n"]+  [ line_ $+    def_ "go" [p_ "n", p_ "acc"]+      [ line_ $+        if_ ("n" .== 0)+          [line_ $ return_ (var_ "acc")] &+        else_+          [line_ . return_ $ call_ "go" [p_ $ "n" .- 1, p_ $ "n" .* "acc"]]+      ]+  , line_ . return_ $ call_ "go" [p_ "n", p_ 1]+  ]++-- |+-- @+-- def spin():+--   spin()+-- @+spin :: Raw Fundef+spin = def_ "spin" [] [line_ $ call_ "spin" []]++-- |+-- @+-- def yes()+--   print("yes")+--   yes()+-- @+yes :: Raw Fundef+yes =+  def_ "yes" []+  [ line_ $ call_ "print" [p_ $ str_ "yes"]+  , line_ $ call_ "yes" []+  ]++counter :: Raw ClassDef+counter =+  class_ "Counter" []+  [ line_ $+    def_ "__init__" ["self"]+      [line_ ("self" /> "x" .= 0)]++  , blank_++  , line_ $+    def_ "incr" ["self"]+      [line_ ("self" /> "x" .+= 1)]++  , blank_++  , line_ $+    def_ "reset" ["self"]+      [line_ ("self" /> "x" .= 0)]++  , blank_++  , line_ $+    def_ "get" ["self"]+      [line_ $ return_ ("self" /> "x")]+  ]++exceptions :: Raw Fundef+exceptions =+  def_ "exceptions" []+  [ line_ $+    tryE_ [line_ pass_] &+      except_ [line_ pass_]+  , blank_++  , line_ $+    tryE_ [line_ pass_] &+      exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_]+  , blank_++  , line_ $+    tryE_ [line_ pass_] &+      exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_] &+      finally_ [line_ pass_]+  , blank_++  , line_ $+    tryE_ [line_ pass_] &+      exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_] &+      else_ [line_ pass_] &+      finally_ [line_ pass_]+  , blank_++  , line_ $ tryF_ [line_ pass_] [line_ pass_]+  , blank_++  , line_ $ tryF_ [line_ pass_] & finally_ [line_ pass_]+  , blank_++  , line_ $+    tryF_ [line_ pass_] [line_ pass_] &+      exceptAs_ (var_ "a" `as_` id_ "b") [line_ pass_] &+      else_ [line_ pass_]+  ]++everything :: Raw Module+everything =+  module_+  [ line_ append_to+  , blank_++  , line_ append_to'+  , blank_++  , line_ fact_tr+  , blank_++  , line_ spin+  , blank_++  , line_ yes+  , blank_++  , line_ counter+  , blank_++  , line_ exceptions+  ]
+ example/Validation.hs view
@@ -0,0 +1,63 @@+{-# language OverloadedStrings #-}+{-# language TypeApplications #-}+module Validation where++import qualified Data.Text.IO as Text++import Language.Python.DSL+import Language.Python.Render+import Language.Python.Validate++good_program :: Raw Module+good_program =+  module_+  [ line_ $+    def_ "a" [p_ "b", p_ "c"]+    [ line_ $ return_ ("b" .+ "c")+    ]+  ]++bad_program :: Raw Module+bad_program =+  module_+  [ line_ $+    def_ "a" [p_ "b", p_ "c"]+    [ line_ $ return_ ("b" .+ "d")+    ]+  ]++doValidating :: IO ()+doValidating = do+  putStrLn "Validating good program:\n"++  {-++  We can render unvalidated programs++  -}+  Text.putStrLn $ showModule good_program++  {-++  Validate the module for indentation, syntax, and scope correctness++  We use the type application specify the error type so that we can Show the+  result++  On success, we get back the same program we put in, but it has a slightly+  different type to indicate that it has been validated++  -}+  print $ validateModuleAll @(ValidationError ()) good_program++  putStrLn ""++  putStrLn "Validating bad program:\n"+  Text.putStrLn $ showModule bad_program++  {-++  On failure, we get back a non-empty list of errors that occurred++  -}+  print $ validateModuleAll @(ValidationError ()) bad_program
+ hpython.cabal view
@@ -0,0 +1,209 @@+-- Initial hpython.cabal generated by cabal init.  For further +-- documentation, see http://haskell.org/cabal/users-guide/++name:                hpython+version:             0.1+synopsis:            Python language tools+description:+  `hpython` provides an abstract syntax tree for Python 3.5, along with a parser,+  printer, and syntax checker. It also contains optics for working with the AST,+  and a DSL for writing Python programs directly in Haskell.+  .+  For a high-level overview of the library, see the @Language.Python@ module.+  .+  For code examples, see the [examples directory on GitHub](https://github.com/qfpl/hpython/tree/master/example).+  .+  For general information about the project, see the [project readme](https://github.com/qfpl/hpython/blob/master/README.md).+license:             BSD3+license-file:        LICENCE+author:              Isaac Elliott+maintainer:          isaace71295@gmail.com+copyright:           Copyright (c) 2017-2018, Commonwealth Scientific and Industrial Research Organisation (CSIRO) ABN 41 687 119 230.+category:            Language+build-type:          Simple+extra-source-files:  ChangeLog.md+                     benchmarks/pypy.py+                     test/files/ansible.py+                     test/files/asyncstatements.py+                     test/files/comments.py+                     test/files/decorators.py+                     test/files/dictcomp.py+                     test/files/django.py+                     test/files/django2.py+                     test/files/imaginary.py+                     test/files/indent_optics_in.py+                     test/files/indent_optics_out.py+                     test/files/indent_optics_in2.py+                     test/files/indent_optics_out2.py+                     test/files/joblib.py+                     test/files/joblib2.py+                     test/files/mypy.py+                     test/files/mypy2.py+                     test/files/numpy.py+                     test/files/numpy2.py+                     test/files/pandas.py+                     test/files/pandas2.py+                     test/files/pypy.py+                     test/files/pypy2.py+                     test/files/regex.py+                     test/files/requests.py+                     test/files/requests2.py+                     test/files/set.py+                     test/files/string.py+                     test/files/sqlalchemy.py+                     test/files/test.py+                     test/files/typeann.py+                     test/files/weird.py+                     test/files/weird2.py++cabal-version:       >=1.10+tested-with:           GHC == 8.0.2+                     , GHC == 8.2.2+                     , GHC == 8.4.4+                     , GHC == 8.6.1+++source-repository    head+  type:              git+  location:          git@github.com/qfpl/hpython.git++flag development+  default: False+  manual: True++library+  exposed-modules:     Data.Type.Set+                     , Data.Validate.Monadic+                     , Language.Python+                     , Language.Python.DSL+                     , Language.Python.Internal.Lexer+                     , Language.Python.Internal.Parse+                     , Language.Python.Internal.Render+                     , Language.Python.Internal.Render.Correction+                     , Language.Python.Internal.Token+                     , Language.Python.Internal.Syntax.IR+                     , Language.Python.Optics+                     , Language.Python.Optics.Indents+                     , Language.Python.Optics.Newlines+                     , Language.Python.Optics.Validated+                     , Language.Python.Parse+                     , Language.Python.Parse.Error+                     , Language.Python.Render+                     , Language.Python.Syntax+                     , Language.Python.Syntax.AugAssign+                     , Language.Python.Syntax.CommaSep+                     , Language.Python.Syntax.Comment+                     , Language.Python.Syntax.Expr+                     , Language.Python.Syntax.Ident+                     , Language.Python.Syntax.Import+                     , Language.Python.Syntax.Module+                     , Language.Python.Syntax.Operator.Binary+                     , Language.Python.Syntax.Operator.Unary+                     , Language.Python.Syntax.ModuleNames+                     , Language.Python.Syntax.Numbers+                     , Language.Python.Syntax.Punctuation+                     , Language.Python.Syntax.Raw+                     , Language.Python.Syntax.Statement+                     , Language.Python.Syntax.Strings+                     , Language.Python.Syntax.Types+                     , Language.Python.Syntax.Whitespace+                     , Language.Python.Validate+                     , Language.Python.Validate.Error+                     , Language.Python.Validate.Scope+                     , Language.Python.Validate.Scope.Error+                     , Language.Python.Validate.Syntax+                     , Language.Python.Validate.Syntax.Error+                     , Language.Python.Validate.Indentation+                     , Language.Python.Validate.Indentation.Error+  build-depends:       base >=4.9 && <5+                     , bifunctors >= 0.1 && < 5.6+                     , bytestring >= 0.10 && < 0.11+                     , digit >=0.7 && < 0.8+                     , dlist >=0.8 && <0.9+                     , lens >= 4 && < 4.18+                     , parsers >= 0.10 && < 0.13+                     , megaparsec >=6.3 && <7+                     , fingertree >=0.1 && <0.2+                     , mtl >= 2.1 && < 2.3+                     , containers >=0.5.7.1 && <0.7+                     , deriving-compat >=0.4 && <0.6+                     , semigroupoids >=5.2.2 && <5.4+                     , text >=1.2 && <1.3+                     , these >=0.7.4 && <0.8+                     , validation >= 1 && < 1.1+                     , parsers-megaparsec >=0.1 && <0.2+  hs-source-dirs:      src+  default-language:    Haskell2010+  ghc-options:         -Wall+                       -Wincomplete-patterns+                       -Wincomplete-record-updates+                       -Wunused-imports+                       -fno-warn-name-shadowing+  if flag(development)+    ghc-options:       -Werror++executable example+  main-is:             Main.hs+  other-modules:       Indentation+                     , FixMutableDefaultArguments+                     , OptimizeTailRecursion+                     , Programs+                     , Validation+  hs-source-dirs:      example+  build-depends:       base >=4.9 && <5, lens, hpython, text+  default-language:    Haskell2010+  ghc-options:         -Wincomplete-patterns+                       -Wincomplete-record-updates+                       -Wunused-imports+  if flag(development)+    ghc-options:       -Werror+++benchmark bench+  main-is:             Main.hs+  type:                exitcode-stdio-1.0+  hs-source-dirs:      benchmarks+  build-depends:       base >=4.9 && <5+                     , hpython+                     , megaparsec >=6.3 && < 7+                     , criterion >= 1 && < 1.6+                     , deepseq+                     , text+                     , validation >= 1 && < 1.1+  default-language:    Haskell2010+  ghc-options:         -Wall+                       -Wincomplete-patterns+                       -Wincomplete-record-updates+                       -Wunused-imports+  if flag(development)+    ghc-options:       -Werror+++test-suite hpython-tests+  main-is:             Main.hs+  type:                exitcode-stdio-1.0+  other-modules:       DSL+                     , Helpers+                     , LexerParser+                     , Optics+                     , Parser+                     , Roundtrip+                     , Scope+                     , Syntax+  hs-source-dirs:      test+  build-depends:       base >=4.9 && <5+                     , filepath+                     , hpython+                     , hedgehog >= 0.5 && < 0.7+                     , lens >= 4 && < 4.18+                     , text >=1.2 && <1.3+                     , megaparsec >=6.3 && < 7+                     , validation >= 1 && < 1.1+  default-language:    Haskell2010+  ghc-options:         -Wall+                       -Wincomplete-patterns+                       -Wincomplete-record-updates+                       -Wunused-imports+                       -fno-warn-name-shadowing+  if flag(development)+    ghc-options:       -Werror
+ src/Data/Type/Set.hs view
@@ -0,0 +1,27 @@+{-# language DataKinds, TypeFamilies, TypeOperators #-}+{-# language FlexibleInstances, MultiParamTypeClasses, PolyKinds #-}++{-|+Module      : Data.Type.Set+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++This module defines some helpful set-like functions for working with type-level lists.+-}++module Data.Type.Set (Nub, Member) where++-- | Remove adjacent equal elements from a type-level list+type family Nub t where+  Nub '[] = '[]+  Nub '[e] = '[e]+  Nub (e ': e ': s) = Nub (e ': s)+  Nub (e ': f ': s) = e ': Nub (f ': s)++-- | Determine whether type @a@ is a member of type-level list @s@+class Member a s where+instance {-# OVERLAPS #-} Member a (a ': s) where+instance {-# OVERLAPPABLE #-} Member a s => Member a (b ': s) where
+ src/Data/Validate/Monadic.hs view
@@ -0,0 +1,60 @@+{-# language GeneralizedNewtypeDeriving #-}+{-# language RankNTypes #-}++{-|+Module      : Data.Validate.Monadic+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Data.Validate.Monadic+  ( ValidateM (ValidateM, unValidateM)+  , runValidateM+  , bindVM+  , liftVM0+  , liftVM1+  , errorVM+  , errorVM1+  )+where++import Data.Functor.Compose (Compose(..))+import Data.Semigroup (Semigroup)+import Data.Validation (Validation(..))++-- | The composition of 'Data.Validation.Validation' with another 'Applicative' functor.+--+-- 'Data.Validation.Validation' is not a 'Monad', and 'ValidateM' is not a monad transformer.+-- It is equipped with a useful bind function, but that function does not have+-- the right type to make 'ValidateM' a 'Monad' (besides which it would break+-- the laws)+newtype ValidateM e m a = ValidateM { unValidateM :: Compose m (Validation e) a }+  deriving (Functor, Applicative)++-- | Unwrap a 'ValidateM'+runValidateM :: ValidateM e m a -> m (Validation e a)+runValidateM = getCompose . unValidateM++-- | Bind into a 'ValidateM'. Note that the first parameter is @m a@, not @ValidateM e m a@.+bindVM :: Monad m => m a -> (a -> ValidateM e m b) -> ValidateM e m b+bindVM m f = ValidateM . Compose $ m >>= getCompose . unValidateM . f++-- | Lift into a succeeding validation+liftVM0 :: (Functor m, Semigroup e) => m a -> ValidateM e m a+liftVM0 m = ValidateM . Compose $ pure <$> m++-- | Run a natural transformation across 'ValidateM' to alter @m@+liftVM1 :: (forall x. m x -> m x) -> ValidateM e m a -> ValidateM e m a+liftVM1 f = ValidateM . Compose . f . getCompose . unValidateM++-- | Lift an error into 'ValidateM'+errorVM :: Applicative m => e -> ValidateM e m a+errorVM = ValidateM . Compose . pure . Failure++-- | Lift an error in an 'Applicative' into 'ValidateM'. This is especially+-- useful if you're using list or 'Data.List.NonEmpty.NonEmpty' to collect errors.+errorVM1 :: (Applicative f, Applicative m) => e -> ValidateM (f e) m a+errorVM1 = errorVM . pure
+ src/Language/Python.hs view
@@ -0,0 +1,40 @@+{-|+Module      : Language.Python+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++@hpython@ provides tools for working with Python source code.++"Language.Python.DSL": A DSL for writing Python programs++"Language.Python.Optics": Optics for working with Python syntax trees++"Language.Python.Parse": Parse Python source into a syntax tree++"Language.Python.Render": Pretty print Python syntax trees++"Language.Python.Syntax": The data structures that represent Python programs, like 'Statement' and 'Expr'++"Language.Python.Validate": Validate aspects of Python syntax trees, like indentation, syntax, or scope++-}++module Language.Python+  ( module Language.Python.DSL+  , module Language.Python.Optics+  , module Language.Python.Parse+  , module Language.Python.Render+  , module Language.Python.Syntax+  , module Language.Python.Validate+  )+where++import Language.Python.DSL+import Language.Python.Optics+import Language.Python.Parse+import Language.Python.Render+import Language.Python.Syntax+import Language.Python.Validate
+ src/Language/Python/DSL.hs view
@@ -0,0 +1,2293 @@+{-|+Module      : Language.Python.DSL+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Passing @[]@ to a function which expects a @['Raw' 'Line']@ is the same as+passing @['line_' 'pass_']@+-}+++{-# language DataKinds #-}+{-# language FlexibleContexts #-}+{-# language MultiParamTypeClasses, FunctionalDependencies, FlexibleInstances #-}+{-# language LambdaCase #-}+{-# language RankNTypes #-}+{-# language RecordWildCards #-}+{-# language TemplateHaskell #-}+{-# language TypeApplications #-}+{-# language TypeFamilies #-}+{-# language UndecidableInstances #-}+module Language.Python.DSL+  ( (&)+  , Raw+  , Module+  , Statement+  , Expr+    -- * Modules+  , module_+    -- * Lines of code+  , blank_+  , AsLine(..)+  , Line(..)+    -- * Identifiers+  , id_+  , Ident(..)+    -- ** Lenses+  , identAnn+  , identValue+  , identWhitespace+    -- * Starred values+  , StarSyntax(..)+  , star_+    -- * Double-starred values+  , DoubleStarSyntax(..)+    -- * @as@ syntax+  , As(..)+    -- * @if@ syntax+  , IfSyntax(..)+    -- * @for@ syntax+  , ForSyntax(..)+    -- * @in@ syntax+  , InSyntax(..), In(..), InList(..)+    -- * @:@ syntax+  , ColonSyntax(..)+    -- * Comprehensions+  , comp_+  , Guard(..)+    -- * Parameters and arguments+    -- ** Parameters+  , Param(..)+  , ParametersSyntax(..)+    -- ** Arguments+  , Arg(..)+  , ArgumentsSyntax(..)+    -- ** Positional+  , PositionalSyntax(..)+  , PositionalParam(..)+  , _PositionalParam+    -- *** Lenses+  , ppAnn+  , ppName+  , ppType+    -- ** Keyword+  , KeywordSyntax(..)+  , KeywordParam(..)+  , _KeywordParam+    -- *** Lenses+  , kpAnn+  , kpName+  , kpType+  , kpEquals+  , kpExpr+    -- * Decorators+  , decorated_+  , DecoratorsSyntax(..)+    -- * Statements+    -- ** @async@+  , AsyncSyntax(..)+    -- ** Block bodies+  , BodySyntax(..)+    -- ** Function definitions+  , def_+  , Fundef(..)+  , mkFundef+    -- *** Lenses+  , fdAnn+  , fdDecorators+  , fdIndents+  , fdAsync+  , fdDefSpaces+  , fdName+  , fdLeftParenSpaces+  , fdParameters+  , fdRightParenSpaces+  , fdReturnType+  , fdBody+    -- ** Class definitions+  , class_+  , ClassDef(..)+  , mkClassDef+    -- *** Lenses+  , cdAnn+  , cdDecorators+  , cdIndents+  , cdClass+  , cdName+  , cdArguments+  , cdBody+    -- ** Assignment+  , chainEq+  , (.=)+  , (.+=)+  , (.-=)+  , (.*=)+  , (.@=)+  , (./=)+  , (.%=)+  , (.&=)+  , (.|=)+  , (.^=)+  , (.<<=)+  , (.>>=)+  , (.**=)+  , (.//=)+    -- ** Exceptions+  , tryE_+  , tryF_+  , ExceptSyntax(..)+  , FinallySyntax(..)+  , TryExcept(..)+  , mkTryExcept+  , TryFinally(..)+  , mkTryFinally+  , ExceptAs(..)+  , AsExceptAs(..)+  , Except(..)+  , mkExcept+  , Finally(..)+  , mkFinally+    -- *** Lenses+  , teAnn+  , teIndents+  , teTry+  , teBody+  , teExcepts+  , teElse+  , teFinally+  , exceptIndents+  , exceptExcept+  , exceptExceptAs+  , exceptBody+  , finallyIndents+  , finallyFinally+  , finallyBody+    -- ** With statements+  , with_+  , withItem_+  , With(..)+  , mkWith+  , AsWithItem(..)+  , WithItem(..)+    -- *** Lenses+  , withAnn+  , withIndents+  , withAsync+  , withWith+  , withItems+  , withBody+    -- ** Flow control+    -- *** 'Else' clauses+    -- | 'If', 'While', 'For', and 'TryExcept' statements can have an 'Else'+    -- component.+    --+    -- 'else_' is considered to be a modifier on these structures.+    --+    -- \-\-\-+    --+    -- 'If' ... 'Else':+    --+    -- >>> if_ false_ [line_ pass_] & else_ [line_ pass_]+    -- if False:+    --     pass+    -- else:+    --     pass+    --+    -- \-\-\-+    --+    -- 'While' ... 'Else':+    --+    -- >>> while_ false_ [line_ pass_] & else_ [line_ pass_]+    -- while False:+    --     pass+    -- else:+    --     pass+    --+    -- \-\-\-+    --+    -- 'For' ... 'Else':+    --+    -- >>> for_ (var_ "a" `in_` [var_ b]) [line_ pass_] & else_ [line_ pass_]+    -- for a in b:+    --     pass+    -- else:+    --     pass+    --+    -- \-\-\-+    --+    -- 'TryExcept' ... 'Else':+    --+    -- >>> tryE_ [line_ pass_] & except_ [line_ pass_] & else_ [line_ pass_]+    -- try:+    --     pass+    -- except:+    --     pass+    -- else:+    --     pass+  , else_+  , ElseSyntax(..)+    -- *** Break+  , break_+    -- *** For loops+    -- | 'For' loops are built using 'for_' syntax:+    --+    -- >>> for_ (var_ "a" `in_` [1, 2, 3]) [line_ (call_ "print" [var_ "a"])]+    -- for a in 1, 2, 3:+    --     print(a)+    --+    -- See also: 'ForSyntax'+  , forSt_+  , For(..)+  , _For+  , mkFor+    -- *** If statements+  , ifThen_+  , elif_+  , If(..)+  , mkIf+  , Elif(..)+  , mkElif+  , Else(..)+  , mkElse+    -- **** Lenses+  , ifAnn+  , ifIndents+  , ifIf+  , ifCond+  , ifBody+  , ifElifs+  , ifElse+  , elifIndents+  , elifElif+  , elifCond+  , elifBody+  , elseIndents+  , elseElse+  , elseBody+    -- *** Pass+  , pass_+    -- *** Return+  , return_+    -- *** While loops+  , while_+  , While(..)+  , mkWhile+    -- **** Lenses+  , whileAnn+  , whileIndents+  , whileWhile+  , whileCond+  , whileBody+    -- * Expressions+  , expr_+  , var_+    -- ** @await@+  , await_+    -- ** @... if ... else ...@+  , ifThenElse_+    -- ** Generators+  , gen_+    -- ** @yield@+  , yield_+    -- ** @yield from ...@+  , yieldFrom_+    -- ** Tuples+  , tuple_+  , Tuple(..)+  , AsTupleItem(..)+  , TupleItem()+    -- ** Function calls+  , call_+  , Call(..)+  , mkCall+    -- *** Lenses+  , callAnn+  , callFunction+  , callLeftParen+  , callArguments+  , callRightParen+    -- ** Literals+    -- *** @None@+  , none_+  , None(..)+  , _None+    -- **** Lenses+  , noneAnn+  , noneWhitespace+    -- *** Strings+  , str_+  , str'_+  , longStr_+  , longStr'_+    -- *** Integers+  , int_+    -- *** Booleans+  , true_+  , false_+    -- *** Ellipses+  , ellipsis_+    -- ** Lists+  , AsList(..)+  , AsListItem(..)+  , ListItem()+    -- ** Dictionaries+  , AsDict(..)+  , DictItem()+    -- ** Sets+  , AsSet(..)+  , AsSetItem(..)+  , SetItem()+    -- ** Lambdas+  , lambda_+    -- ** Subscripting+  , subs_+    -- *** Slicing+  , sliceF_+  , sliceFS_+  , sliceT_+  , sliceTS_+  , sliceFT_+  , sliceFTS_+  , sliceS_+  , fullSlice_+  , slice_+    -- ** Dereferencing+  , (/>)+    -- ** Unary operators+  , not_+  , neg_+  , pos_+  , compl_+    -- ** Binary operators+    -- | Comparison, bitwise, and arithmetic operators have precedences that are+    -- consistent with their Python counterparts. This meansPython expressions can+    -- be translated to kellSyntax with minimal parentheses.+    --+    -- Note: this doesn't apply to unary operators (because kellSyntax doesn't have+    -- unary operators), or the boolean operations 'and_' and 'or_' (because we ran+    -- out of precedence levels)++    -- *** Boolean operations+  , or_+  , and_++    -- *** Comparison operations+  , is_+  , isNot_+  , notIn_+  , (.==)+  , (.>)+  , (.>=)+  , (.<)+  , (.<=)+  , (.!=)+    -- *** Bitwise operations+  , (.|)+  , (.^)+  , (.&)+  , (.<<)+  , (.>>)+    -- *** Arithmetic operations+  , (.-)+  , (.+)+  , (.*)+  , (.@)+  , (./)+  , (.//)+  , (.%)+  , (.**)+    -- * Miscellaneous+  , linesToBlock+  , blockToLines+  )+where++import Control.Applicative ((<|>))+import Control.Lens.Fold ((^..), (^?), folded, lengthOf)+import Control.Lens.Getter ((^.), to)+import Control.Lens.Iso (from)+import Control.Lens.Lens (Lens')+import Control.Lens.Prism (_Right, _Just)+import Control.Lens.Review ((#))+import Control.Lens.Setter ((.~), (<>~), (?~), (%~), Setter', set, over, mapped)+import Control.Lens.TH (makeWrapped)+import Control.Lens.Traversal (Traversal', traverseOf)+import Control.Lens.Tuple (_2)+import Control.Lens.Wrapped (_Wrapped)+import Data.Foldable (toList)+import Data.Function ((&))+import Data.String (fromString)+import Data.List.NonEmpty (NonEmpty(..))+import Data.Maybe (fromMaybe)+import Data.Semigroup ((<>))++import Language.Python.Optics+import Language.Python.Syntax.AugAssign+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Module+import Language.Python.Syntax.Operator.Binary+import Language.Python.Syntax.Operator.Unary+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Raw+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Types+import Language.Python.Syntax.Whitespace++-- | 'Ident' has an 'Data.String.IsString' instance, but when a type class dispatches on+-- an 'Ident' we will run into ambiguity if we try to use @OverloadedStrings@. In these+-- cases we can use 'id_' to provide the extra type information+id_ :: String -> Raw Ident+id_ = fromString++-- | Create a 'Module'+--+-- >>> module_+-- >>> [ line_ $ def_ "a" [] [line_ pass_]+-- >>> , blank_+-- >>> , line_ $ def_ "b" [] [line_ pass_]+-- >>> ]+-- def a():+--     pass+-- <BLANKLINE>+-- def b():+--     pass+module_ :: [Raw Line] -> Raw Module+module_ [] = ModuleEmpty+module_ (a:as) =+  case unLine a of+    Left (bl, nl) -> ModuleBlank bl nl $ module_ as+    Right a -> ModuleStatement a $ module_ as++-- | One or more lines of Python code+newtype Line v a+  = Line+  { unLine :: Either (Blank a, Newline) (Statement v a)+  } deriving (Eq, Show)+makeWrapped ''Line++-- | Create a blank 'Line'+blank_ :: Raw Line+blank_ = Line $ Left (Blank () [] Nothing, LF)++-- | Convert some data to a 'Line'+class AsLine s where+  line_ :: Raw s -> Raw Line++instance AsLine SmallStatement where+  line_ ss =+    Line . Right $ SmallStatement (Indents [] ()) ss++instance AsLine SimpleStatement where+  line_ ss =+    Line . Right . SmallStatement (Indents [] ()) $+    MkSmallStatement ss [] Nothing Nothing (Just LF)++instance AsLine CompoundStatement where+  line_ = Line . Right . CompoundStatement++instance AsLine ClassDef where+  line_ = line_ @Statement . (_ClassDef #)++instance AsLine Fundef where+  line_ = line_ @Statement . (_Fundef #)++instance AsLine If where+  line_ = line_ @Statement . (_If #)++instance AsLine While where+  line_ = line_ @Statement . (_While #)++instance AsLine With where+  line_ = line_ @Statement . (_With #)++instance AsLine Statement where+  line_ = Line . Right++instance AsLine Expr where+  line_ e = line_ $ Expr (e ^. exprAnn) e++instance HasExprs Line where+  _Exprs f (Line a) = Line <$> (_Right._Exprs) f a++instance HasStatements Line where+  _Statements f (Line a) = Line <$> _Right f a++class BodySyntax s where+  -- | A faux-Lens that targets lines in the body of some statement-piece, but+  -- does so \'around\' indentation.+  --+  -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ]+  -- def a ():+  --     pass+  --     pass+  --+  -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] ^. body_+  -- pass+  -- pass+  --+  -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] & body_ .~ [ line_ $ var_ "b" += 1 ]+  -- def a():+  --     b += 1+  --+  -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] & body_ <>~ [ line_ $ var_ "b" += 1 ]+  -- def a():+  --     pass+  --     pass+  --     b += 1+  --+  -- >>> def_ "a" [] [ line_ pass_, line_ pass_ ] & body_ .~ []+  -- def a():+  --     pass+  --+  -- \-\-\-+  --+  -- It's a fake 'Lens' because it violates some of the laws. The most obvious violation is+  -- that setting the 'body_' to the empty list actually sets it to a singleton list containing+  -- 'pass_'. (This is because blocks must contain one or more statements)+  body_ :: Functor f => ([Raw Line] -> f [Raw Line]) -> Raw s -> f (Raw s)+  body :: Lens' (Raw s) (Raw Suite)++class ColonSyntax s t | s -> t, t -> s where+  (.:) :: Raw s -> Raw Expr -> Raw t++infix 0 .:++-- | Constructing dictionary items+--+-- @('.:') :: 'Raw' 'Expr' -> 'Raw' 'Expr' -> 'Raw' 'DictItem'@+instance ColonSyntax Expr DictItem where+  (.:) a = DictItem () a (MkColon [Space])++-- | Function parameter type annotations+--+-- @('.:') :: 'Raw' 'Param' -> 'Raw' 'Expr' -> 'Raw' 'Param'@+--+-- 'star_' can be annotated using '.:', but it will have no effect on the output program,+-- as unnamed starred parameters cannot have type annotations.+--+-- See 'def_'+instance ColonSyntax Param Param where+  (.:) p t = p & paramType_ ?~ (MkColon [Space], t)++-- | Positional parameters/arguments+--+-- @+-- p_ :: 'Raw' 'Expr' -> 'Raw' 'Arg'+-- @+--+-- @+-- p_ :: 'Raw' 'Ident' -> 'Raw' 'Param'+-- @+class PositionalSyntax p v | p -> v, v -> p where+  p_ :: Raw v -> Raw p++-- | See 'def_'+instance StarSyntax Ident Param where+  s_ i = StarParam () [] i Nothing++-- | See 'def_'+instance DoubleStarSyntax Ident Param where+  ss_ i = DoubleStarParam () [] i Nothing++class StarSyntax s t | t -> s where+  s_ :: Raw s -> Raw t++-- | See 'call_'+instance StarSyntax Expr Arg where+  s_ = StarArg () []++-- | See 'call_'+instance DoubleStarSyntax Expr Arg where+  ss_ = DoubleStarArg () []++-- | Keyword parameters/arguments+--+-- @+-- p_ :: 'Raw' 'Expr' -> 'Raw' 'Expr' -> 'Raw' 'Arg'+-- @+--+-- @+-- p_ :: 'Raw' 'Ident' -> 'Raw' 'Expr' -> 'Raw' 'Param'+-- @+class KeywordSyntax p where+  k_ :: Raw Ident -> Raw Expr -> Raw p++-- | Unnamed starred parameter+--+-- >>> def_ "a" [ p_ "b", star_ ] [ line_ pass_ ]+-- def a(b, *):+--     pass+star_ :: Raw Param+star_ = UnnamedStarParam () []++class DoubleStarSyntax s t | t -> s where+  ss_ :: Raw s -> Raw t++-- | See 'dict_'+instance DoubleStarSyntax Expr DictItem where+  ss_ = DictUnpack () []++-- | See 'def_'+instance PositionalSyntax Param Ident where+  p_ i = PositionalParam () i Nothing++-- | See 'def_'+instance KeywordSyntax Param where+  k_ a = KeywordParam () a Nothing []++-- | See 'call_'+instance PositionalSyntax Arg Expr where; p_ = PositionalArg ()++-- | See 'call_'+instance KeywordSyntax Arg where; k_ a = KeywordArg () a []++class ParametersSyntax s where+  -- | A faux-Lens that allows targeting 'Param's in-between existing formatting,+  -- and adding appropriate formatting when extra parameters are introduced.+  --+  -- >>> showStatement myStatement+  -- "def a(b ,  c   ):\n    pass"+  --+  -- >>> showStatement (myStatement & _Fundef.parameters_ .~ [p_ "d", p_ "e"]+  -- "def a(d ,  e   ):\n    pass"+  --+  -- >>> showStatement (myStatement & _Fundef.parameters_ .~ [p_ "d", p_ "e", p_ "f"]+  -- "def a(d ,  e   , f):\n    pass"+  --+  -- \-\-\-+  --+  -- It's not a 'Lens' because repeated 'set's can drop trailing commas, violating+  -- the 'Lens' laws. For example:+  --+  -- >>> someFunction+  -- def a(b, c,):+  --     pass+  --+  -- >>> set parameters_ [var_ "d", var_ "e"] someFunction+  -- def a(d, e,):+  --     pass+  --+  -- >>> set parameters_ [] someFunction+  -- def a():+  --     pass+  --+  -- >>> set parameters_ [var_ "d", var_ "e"] (set parameters_ [] someFunction)+  -- def a(d, e):+  --     pass+  parameters_ :: Functor f => ([Raw Param] -> f [Raw Param]) -> Raw s -> f (Raw s)+  parameters :: Lens' (Raw s) (CommaSep (Raw Param))++class ArgumentsSyntax s where+  setArguments :: [Raw Arg] -> Raw s -> Raw s+  getArguments :: Raw s -> [Raw Arg]++class DecoratorsSyntax s where+  setDecorators :: [Raw Expr] -> Raw s -> Raw s+  getDecorators :: Raw s -> [Raw Expr]+  decorators :: Lens' (Raw s) [Raw Decorator]++decorated_ :: DecoratorsSyntax s => [Raw Expr] -> Raw s -> Raw s+decorated_ = setDecorators++exprsToDecorators :: Indents () -> [Raw Expr] -> [Raw Decorator]+exprsToDecorators is = fmap (\e -> Decorator () is (MkAt []) e Nothing LF [])++instance DecoratorsSyntax Fundef where+  decorators = fdDecorators++  setDecorators new code =+    code+    { _fdDecorators = exprsToDecorators (_fdIndents code) new+    }++  getDecorators code = code ^.. fdDecorators.folded._Exprs++blockToLines :: Raw Block -> [Raw Line]+blockToLines (Block x y z) = fmap (Line . Left) x <> (Line (Right y) : fmap Line z)++mkBody_+  :: Traversal' (Raw s) (Indents ())+  -> Lens' (Raw s) (Raw Suite)+  -> forall f. Functor f => ([Raw Line] -> f [Raw Line]) -> Raw s -> f (Raw s)+mkBody_ gIndents gBody f e =+  (\ls -> e & gBody._Blocks .~ mkNewBlock allIndents ls id) <$> blLines'+  where+    -- | The default indent amount is the indentation level of the first statement+    -- in a block. If the first statement has no indentation, it defaults to 4+    -- spaces.+    defaultIndent =+      fromMaybe+        (Indents [replicate 4 Space ^. from indentWhitespaces] ())+        (e ^? gIndents)++    -- | The number of indentation chunks that precede the lines we're focusing on.+    --+    -- It's one more than @defaultIndent@.+    --+    -- For example, if we're looking at this code, which is inside some larger+    -- context:+    --+    -- @+    --     def a():+    --       pass+    -- @+    --+    -- @defaultIndent@ refers to this part:+    --+    -- @+    --      def a():+    --  ^^^^+    -- @+    --+    -- It's a single chunk. The code body has 2 (= one + 1) chunks:+    --+    -- @+    --     def a():+    --       pass+    -- ^^^^+    -- @+    --+    -- and+    --+    -- @+    --     def a():+    --       pass+    --     ^^+    -- @+    --+    -- So we will need to drop/take two chunks from the beginning of each line in+    -- the body.+    numChunks = lengthOf (indentsValue.folded) defaultIndent + 1++    -- | The lines of the block+    blLines = e ^.. gBody._Blocks.to blockToLines.folded++    -- | The lines of the block, with leading indentation chopped off appropriately+    --+    -- For example:+    --+    -- @+    --   def a():+    --      pass+    --      pass+    -- @+    --+    -- the unprocessed lines are:+    --+    -- @+    --      pass+    --      pass+    -- @+    --+    -- so the processed lines should be:+    --+    -- @+    --   pass+    --   pass+    -- @+    blLines' =+      f $+      over+        (mapped._Wrapped._Right._Indents.indentsValue)+        (drop numChunks)+        blLines++    -- | @defaultNewIndent@ is the amount of indentation that 'new' lines should get.+    -- 'New' lines are only introduced when we set the @[Raw Line]@ to a list longer+    -- than its original value.+    --+    -- @allIndents@ is a list of indentation corresponding to the indents of the old+    -- @[Raw Line]@+    defaultNewIndent :: Indents (); allIndents :: [Indents ()]+    (defaultNewIndent, allIndents) =+      foldr+        (\a (di, as) ->+           maybe+             (di, di : as)+             (\x -> (x, x : as))+             (a ^? to unLine._Right._Indents.to (indentsValue %~ take numChunks)))+        (defaultIndent, [])+        blLines++    -- | @mkNewBlock@ zips the old indentation with the new lines, but if the new+    -- list of lines is longer than the old one then the extra lines at the end+    -- are indented by @defaultNewIndent@+    mkNewBlock+      :: [Indents ()]+      -> [Raw Line]+      -> (Raw Block -> Raw Block)+      -> Raw Block+    mkNewBlock [] [] k =+      k $ Block [] (pass_ & _Indents %~ (defaultNewIndent <>)) []+    mkNewBlock (a:_) [] k =+      k $ Block [] (pass_ & _Indents %~ (a <>)) []+    mkNewBlock [] [b] k =+      k $+      either+        (\w -> Block [w] (pass_ & _Indents %~ (defaultNewIndent <>)) [])+        (\w -> Block [] (w & _Indents %~ (defaultNewIndent <>)) [])+        (unLine b)+    mkNewBlock (a:_) [b] k =+      k $+      either+        (\w -> Block [w] (pass_ & _Indents %~ (a <>)) [])+        (\w -> Block [] (w & _Indents %~ (a <>)) [])+        (unLine b)+    mkNewBlock [] (b:bs) k =+      mkNewBlock [] bs $+      \(Block x y z) ->+        k $+        either+          (\w -> Block (w:x) y z)+          (\w ->+             Block []+               (w & _Indents %~ (defaultNewIndent <>))+               ((Left <$> x) <> (Right y:z)))+          (unLine b)+    mkNewBlock (a:as) (b:bs) k =+      mkNewBlock as bs $+      \(Block x y z) ->+        k $+        either+          (\w -> Block (w:x) y z)+          (\w ->+              Block []+                (w & _Indents %~ (a <>))+                ((Left <$> x) <> (Right y:z)))+          (unLine b)++instance BodySyntax Fundef where+  body = fdBody+  body_ = mkBody_ fdIndents fdBody++instance ParametersSyntax Fundef where+  parameters_ f e = flip (set fdParameters) e . go ps <$> ps'+    where+      ps = e ^. fdParameters+      ps' = f $ toList ps++      go :: CommaSep (Raw Param) -> [Raw Param] -> CommaSep (Raw Param)+      go CommaSepNone [] = CommaSepNone+      go CommaSepNone (x:xs) = listToCommaSep $ x:xs+      go CommaSepOne{} [] = CommaSepNone+      go (CommaSepOne a) [x] =+        CommaSepOne $ x & trailingWhitespace .~ (a ^. trailingWhitespace)+      go (CommaSepOne a) (x:xs) =+        listToCommaSep $ (x & trailingWhitespace .~ (a ^. trailingWhitespace)) :xs+      go CommaSepMany{} [] = CommaSepNone+      go (CommaSepMany a b c) (x:xs) =+        CommaSepMany (x & trailingWhitespace .~ (a ^. trailingWhitespace)) b $ go c xs++  parameters = fdParameters++-- | Create a minimal valid function definition+mkFundef :: Raw Ident -> [Raw Line] -> Raw Fundef+mkFundef name body =+  MkFundef+  { _fdAnn = ()+  , _fdDecorators = []+  , _fdIndents = Indents [] ()+  , _fdAsync = Nothing+  , _fdDefSpaces = pure Space+  , _fdName = name+  , _fdLeftParenSpaces = []+  , _fdParameters = CommaSepNone+  , _fdRightParenSpaces = []+  , _fdReturnType = Nothing+  , _fdBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  }++-- |+-- >>> def_ "f" [p_ "x"] [line_ $ return_ "x"]+-- def f(x):+--     return x+--+-- >>> def_ "f" [p_ "x", k_ "y" 2] [line_ $ return_ "x"]+-- def f(x, y=2):+--     return x+--+-- >>> def_ "f" [p_ "x", k_ "y" 2, s_ "z"] [line_ $ return_ "x"]+-- def f(x, y=2, *z):+--     return x+--+-- >>> def_ "f" [p_ "x", k_ "y" 2, s_ "z", ss_ "w"] [line_ $ return_ "x"]+-- def f(x, y=2, *z, **w)+--     return x+--+-- >>> def_ "f" [p_ "x" .: "String"] [line_ $ return_ "x"]+-- def f(x: String):+--     return x+def_ :: Raw Ident -> [Raw Param] -> [Raw Line] -> Raw Fundef+def_ name params body = (mkFundef name body) { _fdParameters = listToCommaSep params }++-- | Create a minimal valid 'Call'+mkCall :: Raw Expr -> Raw Call+mkCall e =+  MkCall+  { _callAnn = ()+  , _callFunction = e+  , _callLeftParen = []+  , _callArguments = Nothing+  , _callRightParen = []+  }++instance ArgumentsSyntax Call where+  setArguments args code =+    code+    { _callArguments =+        case args of+          [] -> Nothing+          a:as -> Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1'+    }++  getArguments code = _callArguments code ^.. folded.folded++-- |+-- >>> call_ "f" [p_ $ var_ "x"]+-- f(x)+--+-- >>> call_ "f" [p_ $ var_ "x", k_ "y" 2]+-- f(x, y=2)+--+-- >>> call_ "f" [p_ $ var_ "x", k_ "y" 2, s_ "z"]+-- f(x, y=2, *z)+--+-- >>> call_ "f" [p_ $ var_ "x", k_ "y" 2, s_ "z", ss_ "w"]+-- f(x, y=2, *z, **w)+call_ :: Raw Expr -> [Raw Arg] -> Raw Expr+call_ expr args =+  _Call #+  (mkCall expr)+  { _callArguments = +    case args of+      [] -> Nothing+      a:as -> Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1'+  }++-- |+-- >>> return_ (var_ "a")+-- return a+return_ :: Raw Expr -> Raw Statement+return_ e =+  SmallStatement+    (Indents [] ())+    (MkSmallStatement (Return () [Space] $ Just e) [] Nothing Nothing (Just LF))++-- | Turns an 'Expr' into a 'Statement'+--+-- >>> expr_ (int_ 3)+-- 3+expr_ :: Raw Expr -> Raw Statement+expr_ e =+  SmallStatement+    (Indents [] ())+    (MkSmallStatement (Expr () e) [] Nothing Nothing (Just LF))++-- |+-- >>> list_ [li_ $ var_ "a"]+-- [a]+--+-- >>> list_ [s_ $ var_ "a"]+-- [*a]+--+-- >>> list_ [li_ $ var_ "a", s_ $ var_ "b"]+-- [a, *b]+--+-- >>> list_ $ comp_ (var_ "a") (for_ $ var_ "a" `in_` list_ [li_ $ int_ 1, li_ $ int_ 2, li_ $ int_ 3]) [if_ $ var_ "a" .== 2]+-- [a for a in [1, 2, 3] if a == 2]+class AsList s where+  list_ :: s -> Raw Expr++class AsListItem s where+  -- | Create a 'ListItem'+  li_ :: Raw s -> Raw ListItem++instance AsListItem ListItem where+  li_ = id++instance AsListItem Expr where+  li_ = ListItem ()++-- | See 'list_'+instance StarSyntax Expr ListItem where+  s_ = ListUnpack () [] []++instance e ~ Raw ListItem => AsList [e] where+  list_ es = List () [] (listToCommaSep1' es) []++instance e ~ Comprehension Expr => AsList (Raw e) where+  list_ c = ListComp () [] c []++newtype Guard v a = MkGuard { unGuard :: Either (CompFor v a) (CompIf v a) }++class ForSyntax a x | a -> x where+  for_ :: Raw x -> a++-- |+-- @'for_' :: 'Raw' 'In' -> 'Raw' 'CompFor'@+--+-- >>> comp_ (var_ "a") (for_ $ var_ "a" `in_` var_ "b") []+-- a for a in b+instance ForSyntax (Raw CompFor) In where+  for_ (MkIn a b) = CompFor () [Space] a [Space] b++-- |+-- @'for_' :: 'Raw' 'In' -> 'Raw' 'Guard'@+--+-- >>> comp_ (var_ "a") (for_ $ var_ "a" `in_` var_ "b") [for_ $ var_ "c" `in_` var_ "d"]+-- a for a in b for c in d+instance ForSyntax (Raw Guard) In where+  for_ (MkIn a b) = MkGuard . Left $ CompFor () [Space] a [Space] b++class IfSyntax a where+  if_ :: Raw Expr -> a++-- |+-- @'if_' :: 'Raw' 'Expr' -> 'Raw' 'Guard'@+--+-- >>> comp_ (var_ "a") (for_ $ var_ "a" `in_` var_ "b") [if_ $ var_ "c" .== var_ "d"]+-- a for a in b if c == d+instance IfSyntax (Raw Guard) where+  if_ = MkGuard . Right . CompIf () [Space]++-- |+-- >>> set_ []+-- set()+--+-- >>> set_ [si_ $ var_ "a"]+-- {a}+--+-- >>> set_ [s_ $ var_ "a"]+-- {*a}+--+-- >>> set_ [si_ $ var_ "a", s_ $ var_ "b"]+-- {a, *b}+--+-- >>> set_ $ comp_ (var_ "a") (for_ $ var_ "a" `in_` set_ [si_ $ int_ 1, si_ $ int_ 2, si_ $ int_ 3]) [if_ $ var_ "a" .== 2]+-- {a for a in [1, 2, 3] if a == 2}+class AsSet s where+  set_ :: s -> Raw Expr++class AsSetItem s where+  -- | Create a 'SetItem'+  si_ :: Raw s -> Raw SetItem++instance AsSetItem SetItem where+  si_ = id++instance AsSetItem Expr where+  si_ = SetItem ()++-- | See 'set_'+instance StarSyntax Expr SetItem where+  s_ = SetUnpack () [] []++instance e ~ Raw SetItem => AsSet [e] where+  set_ es =+    case es of+      [] -> call_ (var_ "set") []+      a:as -> Set () [] ((a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1') []++instance e ~ Comprehension SetItem => AsSet (Raw e) where+  set_ c = SetComp () [] c []++comp_ :: Raw e -> Raw CompFor -> [Raw Guard] -> Raw (Comprehension e)+comp_ val cfor guards =+  Comprehension ()+    val+    (if null guards+     then cfor+     else cfor & trailingWhitespace .~ [Space])+    (unGuard <$> guards)++-- |+-- >>> gen_ $ comp_ (var_ "a") (for_ $ var_ "a" `in_` list_ [li_ $ int_ 1, li_ $ int_ 2, li_ $ int_ 3]) [if_ $ var_ "a" .== 2]+-- (a for a in [1, 2, 3] if a == 2)+gen_ :: Raw (Comprehension Expr) -> Raw Expr+gen_ = Generator ()++-- |+-- >>> dict_ [var_ "a" .: 1]+-- {a: 1}+--+-- >>> dict_ [ss_ $ var_ "a"]+-- {**a}+--+-- >>> dict_ [var_ "a" .: 1, ss_ $ var_ "b"]+-- {a: 1, **b}+--+-- >>> dict_ $ comp_ (var_ "a" .: 1) (for_ $ var_ "a" `in_` list_ [li_ $ int_ 1, li_ $ int_ 2, li_ $ int_ 3]) [if_ $ var_ "a" .== 2]+-- {a: 1 for a in [1, 2, 3] if a == 2}+class AsDict s where+  dict_ :: s -> Raw Expr++-- |+-- @'dict_' :: ['Raw' 'DictItem'] -> 'Raw' 'Expr'@+instance e ~ Raw DictItem => AsDict [e] where+  dict_ ds =+    Dict ()+    []+    (case ds of+       [] -> Nothing+       a:as -> Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1')+    []++-- |+-- @'dict_' :: 'Raw' ('Comprehension' 'DictItem') -> 'Raw' 'Expr'@+instance e ~ Comprehension DictItem => AsDict (Raw e) where+  dict_ comp = DictComp () [] comp []++mkBinOp :: ([Whitespace] -> BinOp ()) -> Raw Expr -> Raw Expr -> Raw Expr+mkBinOp bop a = BinOp () (a & trailingWhitespace .~ [Space]) (bop [Space])++-- | @a is b@+is_ :: Raw Expr -> Raw Expr -> Raw Expr+is_ = mkBinOp $ Is ()+infixl 1 `is_`++-- |+-- >>> var_ "a" `in_` var_ "b"+-- a in b+data In v a = MkIn (Expr v a) (Expr v a)++-- |+-- >>> var_ "a" `in_` [var_ "b", var_ "c"]+-- a in b, c+data InList v a = MkInList (Expr v a) [Expr v a]++class InSyntax a x | a -> x, x -> a where+  in_ :: Raw Expr -> x -> Raw a+infixl 1 `in_`++-- | @a and b@+--+-- Does not have a precedence+and_ :: Raw Expr -> Raw Expr -> Raw Expr+and_ a = BinOp () (a & trailingWhitespace .~ [Space]) (BoolAnd () [Space])++-- | @a or b@+--+-- Does not have a precedence+or_ :: Raw Expr -> Raw Expr -> Raw Expr+or_ a = BinOp () (a & trailingWhitespace .~ [Space]) (BoolOr () [Space])++-- |+-- >>> var_ "a" `in_` var_ "b"+-- a in b+instance InSyntax Expr (Raw Expr) where+  in_ = mkBinOp $ In ()++-- | See 'for_'+instance e ~ Raw Expr => InSyntax InList [e] where+  in_ = MkInList++-- | @a not in b@+notIn_ :: Raw Expr -> Raw Expr -> Raw Expr+notIn_ = mkBinOp $ NotIn () [Space]+infixl 1 `notIn_`++-- | @a is not b@+isNot_ :: Raw Expr -> Raw Expr -> Raw Expr+isNot_ = mkBinOp $ IsNot () [Space]+infixl 1 `isNot_`++-- | @not a@+not_ :: Raw Expr -> Raw Expr+not_ = Not () [Space]++-- | @a == b@+(.==) :: Raw Expr -> Raw Expr -> Raw Expr+(.==) = mkBinOp $ Eq ()+infixl 1 .==++-- | @a < b@+(.<) :: Raw Expr -> Raw Expr -> Raw Expr+(.<) = mkBinOp $ Lt ()+infixl 1 .<++-- | @a <= b@+(.<=) :: Raw Expr -> Raw Expr -> Raw Expr+(.<=) = mkBinOp $ LtEq ()+infixl 1 .<=++-- | @a > b@+(.>) :: Raw Expr -> Raw Expr -> Raw Expr+(.>) = mkBinOp $ Gt ()+infixl 1 .>++-- | @a >= b@+(.>=) :: Raw Expr -> Raw Expr -> Raw Expr+(.>=) = mkBinOp $ GtEq ()+infixl 1 .>=++-- | @a != b@+(.!=) :: Raw Expr -> Raw Expr -> Raw Expr+(.!=) = mkBinOp $ NotEq ()+infixl 1 .!=++-- | @a | b@+(.|) :: Raw Expr -> Raw Expr -> Raw Expr+(.|) = mkBinOp $ BitOr ()+infixl 2 .|++-- | @a ^ b@+(.^) :: Raw Expr -> Raw Expr -> Raw Expr+(.^) = mkBinOp $ BitXor ()+infixl 3 .^++-- | @a & b@+(.&) :: Raw Expr -> Raw Expr -> Raw Expr +(.&) = mkBinOp $ BitAnd ()+infixl 4 .&++-- | @a << b@+(.<<) :: Raw Expr -> Raw Expr -> Raw Expr +(.<<) = mkBinOp $ ShiftLeft ()+infixl 5 .<<++-- | @a >> b@+(.>>) :: Raw Expr -> Raw Expr -> Raw Expr +(.>>) = mkBinOp $ ShiftRight ()+infixl 5 .>>++-- | @a + b@+(.+) :: Raw Expr -> Raw Expr -> Raw Expr +(.+) = (+)+infixl 6 .+++-- | @a - b@+(.-) :: Raw Expr -> Raw Expr -> Raw Expr +(.-) = (-)+infixl 6 .-++-- | @a * b@+(.*) :: Raw Expr -> Raw Expr -> Raw Expr +(.*) = (*)+infixl 7 .*++-- | @a \@ b@+(.@) :: Raw Expr -> Raw Expr -> Raw Expr+(.@) = mkBinOp $ At ()+infixl 7 .@++-- | @a / b@+(./) :: Raw Expr -> Raw Expr -> Raw Expr+(./) = mkBinOp $ Divide ()+infixl 7 ./++-- | @a // b@+(.//) :: Raw Expr -> Raw Expr -> Raw Expr+(.//) = mkBinOp $ FloorDivide ()+infixl 7 .//++-- | @a % b@+(.%) :: Raw Expr -> Raw Expr -> Raw Expr+(.%) = mkBinOp $ Percent ()+infixl 7 .%++-- | @a ** b@+(.**) :: Raw Expr -> Raw Expr -> Raw Expr+(.**) = mkBinOp $ Exp ()+infixr 8 .**++-- |+-- >>> var_ "a" /> var_ "b"+-- a.b+(/>) :: Raw Expr -> Raw Ident -> Raw Expr+(/>) a = Deref () a []+infixl 9 />++-- | @-a@+neg_ :: Raw Expr -> Raw Expr+neg_ = negate++-- | @+a@+pos_ :: Raw Expr -> Raw Expr+pos_ = UnOp () (Positive () [])++-- | @~a@+compl_ :: Raw Expr -> Raw Expr+compl_ = UnOp () (Complement () [])++-- | Convert a list of 'Line's to a 'Block', giving them 4 spaces of indentation+linesToBlockIndented :: [Raw Line] -> Raw Block+linesToBlockIndented = over _Indents (indentIt $ replicate 4 Space) . linesToBlock++-- | Convert a list of 'Line's to a 'Block', without indenting them+linesToBlock :: [Raw Line] -> Raw Block+linesToBlock = go+  where+    go [] = Block [] pass_ []+    go [y] =+      case unLine y of+        Left l -> Block [l] pass_ []+        Right st -> Block [] st []+    go (y:ys) =+      case unLine y of+        Left l ->+          case go ys of+            Block a b c -> Block (l:a) b c+        Right st -> Block [] st (unLine <$> ys)++instance BodySyntax While where+  body = whileBody+  body_ = mkBody_ whileIndents whileBody++instance ElseSyntax While where+  getElse = mkGetElse _whileIndents _whileElse+  setElse = mkSetElse _whileIndents whileElse++-- | Create a minimal valid 'While'+mkWhile :: Raw Expr -> [Raw Line] -> Raw While+mkWhile cond body =+  MkWhile+  { _whileAnn = ()+  , _whileIndents = Indents [] ()+  , _whileWhile = [Space]+  , _whileCond = cond+  , _whileBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  , _whileElse = Nothing+  }++while_ :: Raw Expr -> [Raw Line] -> Raw While+while_ = mkWhile++-- | Create a minimal valid 'If'+mkIf :: Raw Expr -> [Raw Line] -> Raw If+mkIf cond body =+  MkIf+  { _ifAnn = ()+  , _ifIndents = Indents [] ()+  , _ifIf = [Space]+  , _ifCond = cond+  , _ifBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  , _ifElifs = []+  , _ifElse = Nothing+  }++instance BodySyntax Elif where+  body = elifBody+  body_ = mkBody_ elifIndents elifBody ++instance BodySyntax Else where+  body = elseBody+  body_ = mkBody_ elseIndents elseBody ++instance BodySyntax If where+  body = ifBody+  body_ = mkBody_ ifIndents ifBody ++-- |+-- @'if_' :: 'Raw' 'Expr' -> ['Raw' 'Line'] -> 'Raw' 'If'@+--+-- >>> if_ (var_ "a" .< 10) [var_ "a" .+= 1]+-- if a < 10:+--     a += 1+instance (l ~ Raw Line, s ~ Raw If) => IfSyntax ([l] -> s) where+  if_ = mkIf++ifThen_ :: Raw Expr -> [Raw Line] -> Raw If+ifThen_ = mkIf++var_ :: String -> Raw Expr+var_ s = Ident $ MkIdent () s []++-- |+-- >>> none_+-- None+none_ :: Raw Expr+none_ = None () []++-- | @'Raw' 'Expr'@ has a 'Num' instance, but sometimes we need to name integers+-- explicitly+--+-- >>> int_ 10+-- 10+int_ :: Integer -> Raw Expr+int_ = fromInteger++-- |+-- >>> pass_+-- pass+pass_ :: Raw Statement+pass_ =+  SmallStatement+    (Indents [] ())+    (MkSmallStatement (Pass () []) [] Nothing Nothing (Just LF))++-- | Create a minimal valid 'Elif'+mkElif :: Raw Expr -> [Raw Line] -> Raw Elif+mkElif cond body =+  MkElif+  { _elifIndents = Indents [] ()+  , _elifElif = [Space]+  , _elifCond = cond+  , _elifBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  }++elif_ :: Raw Expr -> [Raw Line] -> Raw If -> Raw If+elif_ cond body code = code & ifElifs <>~ [mkElif cond body]++-- | Create a minimal valid 'Else'+mkElse :: [Raw Line] -> Raw Else+mkElse body =+  MkElse+  { _elseIndents = Indents [] ()+  , _elseElse = []+  , _elseBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  }++class ElseSyntax s where+  getElse :: Raw s -> Maybe (Raw Else)+  setElse :: [Whitespace] -> Maybe (Raw Else) -> Raw s -> Raw s++else_ :: ElseSyntax s => [Raw Line] -> Raw s -> Raw s+else_ body = setElse (replicate 4 Space) $ Just (mkElse body)++mkGetElse+  :: (Raw s -> Indents ())+  -> (Raw s -> Maybe (Raw Else))+  -> Raw s+  -> Maybe (Raw Else)+mkGetElse indentLevel elseField code =+  fromMaybe+    (error "malformed indentation in else block")+    (traverseOf+        (traverse._Indents)+        (subtractStart (indentLevel code))+        (elseField code))++mkSetElse+  :: (Raw s -> Indents ())+  -> Setter' (Raw s) (Maybe (Raw Else))+  -> [Whitespace]+  -> Maybe (Raw Else)+  -> Raw s+  -> Raw s+mkSetElse indentLevel elseField _ new code =+  code &+  elseField .~+    fmap (elseIndents .~ indentLevel code)+    (over+       (traverse._Indents.indentsValue)+       (indentLevel code ^. indentsValue <>)+       new)++instance ElseSyntax For where+  getElse = mkGetElse _forIndents _forElse+  setElse = mkSetElse _forIndents forElse++instance ElseSyntax If where+  getElse = mkGetElse _ifIndents _ifElse+  setElse = mkSetElse _ifIndents ifElse++instance ElseSyntax TryExcept where+  getElse = mkGetElse _teIndents _teElse+  setElse = mkSetElse _teIndents teElse++break_ :: Raw Statement+break_ =+  SmallStatement+    (Indents [] ())+    (MkSmallStatement (Break () []) [] Nothing Nothing (Just LF))++-- |+-- >>> true_+-- True+true_ :: Raw Expr+true_ = Bool () True []++-- |+-- >>> false_+-- False+false_ :: Raw Expr+false_ = Bool () False []++-- | Double-quoted string+--+-- >>> str_ "asdf"+-- "asdf"+str_ :: String -> Raw Expr+str_ s =+  String () . pure $+  StringLiteral () Nothing ShortString DoubleQuote (Char_lit <$> s) []++-- | Single-quoted string+--+-- >>> str_ "asdf"+-- 'asdf'+str'_ :: String -> Raw Expr+str'_ s =+  String () . pure $+  StringLiteral () Nothing ShortString SingleQuote (Char_lit <$> s) []++-- | Long double-quoted string+--+-- >>> longStr_ "asdf"+-- """asdf"""+longStr_ :: String -> Raw Expr+longStr_ s =+  String () . pure $+  StringLiteral () Nothing LongString DoubleQuote (Char_lit <$> s) []++-- | Long single-quoted string+--+-- >>> longStr'_ "asdf"+-- '''asdf'''+longStr'_ :: String -> Raw Expr+longStr'_ s =+  String () . pure $+  StringLiteral () Nothing LongString SingleQuote (Char_lit <$> s) []++mkAugAssign :: AugAssignOp -> Raw Expr -> Raw Expr -> Raw Statement+mkAugAssign at a b =+  SmallStatement+    (Indents [] ())+    (MkSmallStatement+       (AugAssign () (a & trailingWhitespace .~ [Space]) (MkAugAssign at () [Space]) b)+       []+       Nothing+       Nothing+       (Just LF))++-- | Chained assignment+--+-- >>> chainEq (var_ "a") []+-- a+--+-- >>> chainEq (var_ "a") [var_ "b", var_ "c"]+-- a = b = c+chainEq :: Raw Expr -> [Raw Expr] -> Raw Statement+chainEq t [] = expr_ t+chainEq t (a:as) =+  SmallStatement+    (Indents [] ())+    (MkSmallStatement+       (Assign () t $ (,) (MkEquals [Space]) <$> (a :| as))+       []+       Nothing+       Nothing+       (Just LF))++-- | @a = b@+(.=) :: Raw Expr -> Raw Expr -> Raw Statement+(.=) a b =+  SmallStatement+    (Indents [] ())+    (MkSmallStatement+       (Assign () (a & trailingWhitespace .~ [Space]) $ pure (MkEquals [Space], b))+       []+       Nothing+       Nothing+       (Just LF))+infix 0 .=++-- | @a += b@+(.+=) :: Raw Expr -> Raw Expr -> Raw Statement+(.+=) = mkAugAssign PlusEq+infix 0 .+=++-- | @a -= b@+(.-=) :: Raw Expr -> Raw Expr -> Raw Statement+(.-=) = mkAugAssign MinusEq+infix 0 .-=++-- | @a *= b@+(.*=) :: Raw Expr -> Raw Expr -> Raw Statement+(.*=) = mkAugAssign StarEq+infix 0 .*=++-- | @a @= b@+(.@=) :: Raw Expr -> Raw Expr -> Raw Statement+(.@=) = mkAugAssign AtEq+infix 0 .@=++-- | @a /= b@+(./=) :: Raw Expr -> Raw Expr -> Raw Statement+(./=) = mkAugAssign SlashEq+infix 0 ./=++-- | @a %= b@+(.%=) :: Raw Expr -> Raw Expr -> Raw Statement+(.%=) = mkAugAssign PercentEq+infix 0 .%=++-- | @a &= b@+(.&=) :: Raw Expr -> Raw Expr -> Raw Statement+(.&=) = mkAugAssign AmpersandEq+infix 0 .&=++-- | @a |= b@+(.|=) :: Raw Expr -> Raw Expr -> Raw Statement+(.|=) = mkAugAssign PipeEq+infix 0 .|=++-- | @a ^= b@+(.^=) :: Raw Expr -> Raw Expr -> Raw Statement+(.^=) = mkAugAssign CaretEq+infix 0 .^=++-- | @a <<= b@+(.<<=) :: Raw Expr -> Raw Expr -> Raw Statement+(.<<=) = mkAugAssign ShiftLeftEq+infix 0 .<<=++-- | @a >>= b@+(.>>=) :: Raw Expr -> Raw Expr -> Raw Statement+(.>>=) = mkAugAssign ShiftRightEq+infix 0 .>>=++-- | @a **= b@+(.**=) :: Raw Expr -> Raw Expr -> Raw Statement+(.**=) = mkAugAssign DoubleStarEq+infix 0 .**=++-- | @a //= b@+(.//=) :: Raw Expr -> Raw Expr -> Raw Statement+(.//=) = mkAugAssign DoubleSlashEq+infix 0 .//=++mkFor :: Raw Expr -> [Raw Expr] -> [Raw Line] -> Raw For+mkFor binder collection body =+  MkFor+  { _forAnn = ()+  , _forIndents = Indents [] ()+  , _forAsync = Nothing+  , _forFor = [Space]+  , _forBinder = binder & trailingWhitespace .~ [Space]+  , _forIn = [Space]+  , _forCollection =+      fromMaybe+        (CommaSepOne1' (Unit () [] []) Nothing)+        (listToCommaSep1' collection)+  , _forBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  , _forElse = Nothing+  }++-- |+-- @'for_' :: 'Raw' 'InList' -> ['Raw' 'Line'] -> 'Raw' 'Statement'@+--+-- >>> for_ (var_ "a" `in_` [var_ "b"]) [line_ (var_ "c" .+= var_ "a")]+-- for a in b:+--     c += a+instance (l ~ [Raw Line], s ~ Raw For) => ForSyntax (l -> s) InList where+  for_ (MkInList a b) = mkFor a b++forSt_ :: Raw Expr -> [Raw Expr] -> [Raw Line] -> Raw For+forSt_ = mkFor++instance BodySyntax For where+  body = forBody+  body_ = mkBody_ forIndents forBody++instance AsLine For where+  line_ = line_ @Statement . (_For #)++class AsyncSyntax s where+  async_ :: Raw s -> Raw s++instance AsyncSyntax Fundef where+  async_ = fdAsync ?~ pure Space++instance AsyncSyntax For where+  async_ = forAsync ?~ pure Space++-- | Create a minimal valid 'Finally'+mkFinally :: [Raw Line] -> Raw Finally+mkFinally body =+  MkFinally+  { _finallyIndents = Indents [] ()+  , _finallyFinally = []+  , _finallyBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  }++-- | Create a minimal valid 'Except'+mkExcept :: [Raw Line] -> Raw Except+mkExcept body =+  MkExcept+  { _exceptIndents = Indents [] ()+  , _exceptExcept = []+  , _exceptExceptAs = Nothing+  , _exceptBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  }++-- | Create a minimal valid 'TryExcept'+mkTryExcept :: [Raw Line] -> Raw Except -> Raw TryExcept+mkTryExcept body except =+  MkTryExcept+  { _teAnn = ()+  , _teIndents = Indents [] ()+  , _teTry = [Space]+  , _teBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  , _teExcepts = pure except+  , _teElse = Nothing+  , _teFinally = Nothing+  }++-- | Create a minimal valid 'TryFinally'+mkTryFinally :: [Raw Line] -> [Raw Line] -> Raw TryFinally+mkTryFinally body fBody =+  MkTryFinally+  { _tfAnn = ()+  , _tfIndents = Indents [] ()+  , _tfTry = [Space]+  , _tfBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  , _tfFinally = mkFinally fBody+  }++class FinallySyntax s t | s -> t where+  finally_ :: [Raw Line] -> s -> Raw t++-- |+-- >>> tryE_ [line_ pass_] & finally_ [line_ pass_]+-- try:+--     pass+-- finally:+--     pass+--+-- >>> tryF_ [line_ pass_] [line_ (a .+= 1)] & finally_ [line_ pass_]+-- try:+--     pass+-- finally:+--     pass+--+-- >>> tryF_ [line_ pass_] & finally_ [line_ pass_]+-- try:+--     pass+-- finally:+--     pass+instance FinallySyntax (Raw TryExcept) TryExcept where+  finally_ body = teFinally ?~ mkFinally body++instance FinallySyntax (Raw TryFinally) TryFinally where+  finally_ body = tfFinally .~ mkFinally body++instance (a ~ [Raw Line], b ~ Raw TryFinally) => FinallySyntax (a -> b) TryFinally where+  finally_ body f = f body++instance BodySyntax TryExcept where+  body = teBody+  body_ = mkBody_ teIndents teBody++-- | @try ... except@ with optional @else@ and optional @finally@+--+-- >>> tryE_ [line_ pass_] [line_ ("a" .+= 1)]+-- try:+--     pass+-- except+--     a += 1+tryE_ :: [Raw Line] -> Raw Except -> Raw TryExcept+tryE_ = mkTryExcept++instance BodySyntax TryFinally where+  body = tfBody+  body_ = mkBody_ tfIndents tfBody ++-- |+-- @try ... finally@+--+-- >>> tryF_ [line_ pass_] [line_ ("a" .+= 1)]+-- try:+--     pass+-- finally:+--     a += 1+tryF_ :: [Raw Line] -> [Raw Line] -> Raw TryFinally+tryF_ = mkTryFinally++class AsExceptAs s where+  toExceptAs :: Raw s -> Raw ExceptAs++instance AsExceptAs ExceptAs where+  toExceptAs = id++instance AsExceptAs Expr where+  toExceptAs e = ExceptAs () e Nothing++class ExceptSyntax s where+  except_ :: [Raw Line] -> s -> Raw TryExcept+  -- | You can use 'exceptAs_' without a binder:+  --+  -- @'exceptAs_' :: 'Raw' 'Expr' -> ['Raw' 'Line'] -> 'Raw' s -> 'Raw' 'TryExcept'@+  --+  -- @+  -- 'exceptAs_' ('var_' \"Exception\") body+  -- @+  --+  -- or with a binder:+  --+  -- @'exceptAs_' :: 'Raw' 'ExceptAs' -> ['Raw' 'Line'] -> 'Raw' s -> 'Raw' 'TryExcept'@+  --+  -- @+  -- 'exceptAs_' ('var_' \"Exception\" \``as_`\` 'id_' "a") body+  -- @+  exceptAs_ :: AsExceptAs e => Raw e -> [Raw Line] -> s -> Raw TryExcept++-- |+-- @'except_' :: ['Raw' 'Line'] -> ('Raw' 'Except' -> 'Raw' 'TryExcept') -> 'Raw' 'TryExcept'@+--+-- @'exceptAs_' :: 'AsExceptAs' e => 'Raw' e -> ['Raw' 'Line'] -> ('Raw' 'Except' -> 'Raw' 'TryExcept') -> 'Raw' 'TryExcept'@+--+-- >>> tryE_ [var_ "a" .= 2] & except_ [var_ "a" .= 3]+-- try:+--     a = 2+-- except:+--     a = 3+--+-- >>> tryE_ [var_ "a" .= 2] & exceptAs_ (var_ "Exception" `as_` id_ "b") [var_ "a" .= 3]+-- try:+--     a = 2+-- except Exception as b:+--     a = 3+instance (e ~ Raw Except, s ~ Raw TryExcept) => ExceptSyntax (e -> s) where+  except_ body f = f $ mkExcept body+  exceptAs_ ea body f = f $ mkExcept body & exceptExceptAs ?~ toExceptAs ea++-- |+-- @'except_' :: ['Raw' 'Line'] -> 'Raw' 'TryExcept' -> 'Raw' 'TryExcept'@+--+-- @'exceptAs_' :: AsExceptAs => 'Raw' e -> ['Raw' 'Line'] -> 'Raw' 'TryExcept' -> 'Raw' 'TryExcept'@+--+-- @+-- (someTryStatement :: 'Raw' 'TryExcept') '&'+--   'except_' ['line_' 'pass_']+-- @+--+-- @+-- (someTryStatement :: 'Raw' 'TryExcept') '&'+--   'exceptAs_' ('var_' \"Exception\" \``as_`\` 'id_' "b") ['line_' 'pass_']+-- @+instance ExceptSyntax (Raw TryExcept) where+  except_ body = teExcepts %~ (<> pure (mkExcept body))+  exceptAs_ ea body =+    teExcepts %~ (<> pure (mkExcept body & exceptExceptAs ?~ toExceptAs ea))++-- |+-- @'except_' :: ['Raw' 'Line'] -> 'Raw' 'TryFinally' -> 'Raw' 'TryExcept'@+--+-- @'exceptAs_' :: AsExceptAs => 'Raw' e -> ['Raw' 'Line'] -> 'Raw' 'TryFinally' -> 'Raw' 'TryExcept'@+--+-- @+-- (someTryStatement :: 'Raw' 'TryFinally') '&'+--   'except_' ['line_' 'pass_']+-- @+--+-- @+-- (someTryStatement :: 'Raw' 'TryFinally') '&'+--   'exceptAs_' ('var_' \"Exception\" \``as_`\` 'id_' "b") ['line_' 'pass_']+-- @+instance ExceptSyntax (Raw TryFinally) where+  except_ body MkTryFinally{..} =+    MkTryExcept+    { _teAnn = _tfAnn+    , _teIndents = _tfIndents+    , _teTry = _tfTry+    , _teBody = _tfBody+    , _teExcepts = pure $ mkExcept body+    , _teElse = Nothing+    , _teFinally = Just _tfFinally+    }++  exceptAs_ ea body MkTryFinally{..} =+    MkTryExcept+    { _teAnn = _tfAnn+    , _teIndents = _tfIndents+    , _teTry = _tfTry+    , _teBody = _tfBody+    , _teExcepts = pure $ mkExcept body & exceptExceptAs ?~ toExceptAs ea+    , _teElse = Nothing+    , _teFinally = Just _tfFinally+    }++instance AsLine TryExcept where+  line_ = line_ @Statement . (_TryExcept #)++instance AsLine TryFinally where+  line_ = line_ @Statement . (_TryFinally #)++class As s t u | s t -> u, u -> s t where+  as_ :: Raw s -> Raw t -> Raw u++-- | See 'exceptAs_'+instance As Expr Ident ExceptAs where+  as_ e name = ExceptAs () e $ Just ([Space], name)++-- |+-- >>> class_ "A" [] [line_ pass_]+-- class A:+--     pass+class_ :: Raw Ident -> [Raw Arg] -> [Raw Line] -> Raw ClassDef+class_ name args body =+  (mkClassDef name body) {+    _cdArguments =+      case args of+        [] -> Nothing+        a:as -> Just ([], Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1', [])+  }++-- | Create a minimal 'ClassDef'+mkClassDef :: Raw Ident -> [Raw Line] -> Raw ClassDef+mkClassDef name body =+  MkClassDef+  { _cdAnn = ()+  , _cdDecorators = []+  , _cdIndents = Indents [] ()+  , _cdClass = Space :| []+  , _cdName = name+  , _cdArguments = Nothing+  , _cdBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  }++instance BodySyntax ClassDef where+  body = cdBody+  body_ = mkBody_ cdIndents cdBody ++instance DecoratorsSyntax ClassDef where+  decorators = cdDecorators++  setDecorators new code =+    code+    { _cdDecorators = exprsToDecorators (_cdIndents code) new+    }++  getDecorators code = code ^.. cdDecorators.folded._Exprs++instance ArgumentsSyntax ClassDef where+  setArguments args code =+    code+    { _cdArguments =+        case args of+          [] -> Nothing+          a:as -> Just ([], Just $ (a, zip (repeat (MkComma [Space])) as, Nothing) ^. _CommaSep1', [])+    }++  getArguments code = _cdArguments code ^.. folded._2.folded.folded++-- | Create a minimal valid 'With'+mkWith :: NonEmpty (Raw WithItem) -> [Raw Line] -> Raw With+mkWith items body =+  MkWith+  { _withAnn = ()+  , _withIndents = Indents [] ()+  , _withAsync = Nothing+  , _withWith = [Space]+  , _withItems = listToCommaSep1 items+  , _withBody = SuiteMany () (MkColon []) Nothing LF $ linesToBlockIndented body+  }++-- |+--+-- @+-- with_ :: 'NonEmpty' ('Raw' 'Expr') -> ['Raw' 'Line'] -> 'Raw' 'Statement'+-- with_ :: 'NonEmpty' ('Raw' 'WithItem') -> ['Raw' 'Line'] -> 'Raw' 'Statement'+-- @+--+-- >>> with_ [var_ "a"] [line_ $ var_ "b"]+-- with a:+--     b+--+-- >>> with_ [var_ "a" `as_` id_ "name"] [line_ $ var_ "b"]+-- with a as name:+--     b+--+-- >>> with_ [withItem_ e Nothing] [line_ $ var_ "b"]+-- with a:+--     b+with_ :: AsWithItem e => NonEmpty (Raw e) -> [Raw Line] -> Raw With+with_ items = mkWith (toWithItem <$> items)++withItem_ :: Raw Expr -> Maybe (Raw Expr) -> Raw WithItem+withItem_ a b = WithItem () a ((,) [Space] <$> b)++-- | See 'with_'+instance As Expr Expr WithItem where+  as_ a b = WithItem () a $ Just ([Space], b)++class AsWithItem s where+  toWithItem :: Raw s -> Raw WithItem++instance AsWithItem Expr where+  toWithItem e = WithItem () e Nothing++instance AsWithItem WithItem where+  toWithItem = id++instance BodySyntax With where+  body = withBody+  body_ = mkBody_ withIndents withBody ++instance AsyncSyntax With where+  async_ = withAsync ?~ pure Space++-- |+-- >>> ellipsis_+-- ...+ellipsis_ :: Raw Expr+ellipsis_ = Ellipsis () []++class AsTupleItem e where+  -- | Create a 'TupleItem'+  ti_ :: Raw e -> Raw TupleItem++-- | See 'tuple_'+instance StarSyntax Expr TupleItem where+  s_ = TupleUnpack () [] []++instance AsTupleItem Expr where+  ti_ = TupleItem ()++instance AsTupleItem TupleItem where+  ti_ = id++-- |+-- >>> tuple_ []+-- ()+--+-- >>> tuple_ [ti_ $ var_ "a"]+-- a,+--+-- >>> tuple_ [s_ $ var_ "a"]+-- (*a),+--+-- >>> tuple_ [ti_ $ var_ "a", ti_ $ var_ "b"]+-- a, b+--+-- >>> tuple_ [ti_ $ var_ "a", s_ $ var_ "b"]+-- a, *b+tuple_ :: [Raw TupleItem] -> Raw Expr+tuple_ [] = Unit () [] []+tuple_ (a:as) =+  case as of+    [] -> Tuple () (ti_ a) (MkComma []) Nothing+    b:bs ->+      Tuple () a (MkComma [Space]) . Just $+      (b, zip (repeat (MkComma [Space])) bs, Nothing) ^. _CommaSep1'++-- |+-- >>> await (var_ "a")+-- await a+await_ :: Raw Expr -> Raw Expr+await_ = Await () [Space]++-- |+-- >>> ifThenElse_ (var_ "a") (var_ "b") (var_ "c")+-- a if b else c+ifThenElse_ :: Raw Expr -> Raw Expr -> Raw Expr -> Raw Expr+ifThenElse_ a b = Ternary () a [Space] b [Space]++-- |+-- >>> lambda_ [p_ "x"] "x"+-- lambda x: x+--+-- >>> lambda_ [p_ "x", k_ "y" 2] ("x" .+ "y")+-- lambda x, y=2: x + y+--+-- >>> lambda_ [p_ "x", k_ "y" 2, s_ "z"] "a"+-- lambda x, y=2, *z: a+--+-- >>> lambda_ [p_ "x", k_ "y" 2, s_ "z", ss_ "w"] "a"+-- lambda x, y=2, *z, **w: a+lambda_ :: [Raw Param] -> Raw Expr -> Raw Expr+lambda_ params =+  Lambda ()+    (if null params then [] else [Space])+    (listToCommaSep params)+    (MkColon [Space])++-- |+-- >>> yield_ []+-- yield+--+-- >>> yield_ [var_ "a"]+-- yield a+--+-- >>> yield_ [var_ "a", var_ "b"]+-- yield a, b+yield_ :: [Raw Expr] -> Raw Expr+yield_ as = Yield () (foldr (\_ _ -> [Space]) [] as) (listToCommaSep as)++-- |+-- >>> yieldFrom_ (var_ "a")+-- yield from a+yieldFrom_ :: Raw Expr -> Raw Expr+yieldFrom_ = YieldFrom () [Space] [Space]++-- | The slice with no bounds+--+-- >>> subs_ (var_ "a") fullSlice_+-- a[:]+--+-- >>> fullSlice_+-- slice(None, None, None)+fullSlice_ :: Raw Expr+fullSlice_ = slice_ Nothing Nothing Nothing++-- | Slice with *step* @x@+--+-- >>> subs_ (var_ "a") (sliceS_ $ int_ (-1))+-- a[::-1]+--+-- >>> sliceS_ $ int_ (-1)+-- slice(None, None, -1)+sliceS_ :: Raw Expr -> Raw Expr+sliceS_ x = slice_ Nothing Nothing (Just x)++-- | Slice *from* @x@+--+-- >>> subs_ (var_ "a") (sliceF_ $ int_ 0)+-- a[1:]+--+-- >>> sliceF_ $ int_ 0+-- slice(1, None, None)+sliceF_ :: Raw Expr -> Raw Expr+sliceF_ x = slice_ (Just x) Nothing Nothing++-- | Slice *from* @x@, with *step* @y@+--+-- >>> subs_ (var_ "a") (sliceFS_ (int_ 0) (int_ 2))+-- a[1::2]+--+-- >>> sliceFS_ (int_ 0) (int_ 2)+-- slice(1, None, 2)+sliceFS_ :: Raw Expr -> Raw Expr -> Raw Expr+sliceFS_ x y = slice_ (Just x) Nothing (Just y)++-- | Slice /To/ @x@+--+-- >>> subs_ (var_ "a") (sliceT_ $ int_ 10)+-- a[:10]+--+-- >>> sliceT_ $ int_ 10+-- slice(None, 10, None)+sliceT_ :: Raw Expr -> Raw Expr+sliceT_ x = slice_ Nothing (Just x) Nothing++-- | Slice /To/ @x@, with /Step/ @y@+--+-- >>> subs_ (var_ "a") (sliceTS_ (int_ 10) (int_ 2))+-- a[:10:2]+--+-- >>> sliceTS_ (int_ 10) (int_ 2)+-- slice(None, 10, 2)+sliceTS_ :: Raw Expr -> Raw Expr -> Raw Expr+sliceTS_ x y = slice_ Nothing (Just x) (Just y)++-- | Slice /From/ @x@ /To/ @y@+--+-- >>> subs_ (var_ "a") (sliceFT_ (int_ 1) (int_ 10))+-- a[1:10]+--+-- >>> sliceFT_ (int_ 1) (int_ 10)+-- slice(1, 10, None)+sliceFT_ :: Raw Expr -> Raw Expr -> Raw Expr+sliceFT_ x y = slice_ (Just x) (Just y) Nothing++-- | Slice /From/ @x@ /To/ @y@, with /Step/ @z@+--+-- >>> subs_ (var_ "a") (sliceFTS_ (int_ 1) (int_ 10) (int_ 2))+-- a[1:10:2]+--+-- >>> sliceFTS_ (int_ 1) (int_ 10) (int_ 2)+-- slice(1, 10, 2)+sliceFTS_ :: Raw Expr -> Raw Expr -> Raw Expr -> Raw Expr+sliceFTS_ x y z = slice_ (Just x) (Just y) (Just z)++-- | A slice object+--+-- Represents a call to a function named @slice@, with 3 arguments.+-- If an argument is a 'Nothing' then it becomes 'None', and if the argument is a+-- 'Just' then the contents are extracted.+slice_ :: Maybe (Raw Expr) -> Maybe (Raw Expr) -> Maybe (Raw Expr) -> Raw Expr+slice_ a b c =+  call_ (var_ "slice")+    [ p_ $ fromMaybe none_ a+    , p_ $ fromMaybe none_ b+    , p_ $ fromMaybe none_ c+    ]++-- |+-- >>> subs_ (var_ "a") (int_ 1)+-- a[1]+--+-- >>> subs_ (var_ "a") (tuple_ [ti_ $ int_ 1])+-- a[1,]+--+-- >>> subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, ti_ $ int_ 2])+-- a[1, 2]+--+-- >>> subs_ (var_ "a") (tuple_ [s_ $ var_ "b"])+-- a[((*b),)]+--+-- >>> subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, s_ $ var_ "b"])+-- a[(1, *b)]+subs_ :: Raw Expr -> Raw Expr -> Raw Expr+subs_ a e =+  Subscript () a+    []+    (exprToSubscript e ^. _CommaSep1')+    []+  where+    exprToSubscript+      :: Raw Expr+      -> (Raw Subscript, [(Comma, Raw Subscript)], Maybe Comma)+    exprToSubscript e =+      let+        notSlice :: (Raw Subscript, [(Comma, Raw Subscript)], Maybe Comma)+        notSlice =+          case e ^? _Tuple of+            Nothing -> (SubscriptExpr e, [], Nothing)+            Just tup ->+              let+                h = tup ^. tupleHead+                comma = tup ^. tupleComma+                t = tup ^? tupleTail._Just.from _CommaSep1'+                res =+                  case t of+                    Just (a, bs, c) ->+                      (,,) <$>+                      fromTupleItem h <*>+                      traverseOf (traverse._2) fromTupleItem ((comma, a) : bs) <*>+                      pure c+                    Nothing -> (\a -> (a, [], Just comma)) <$> fromTupleItem h+              in+                fromMaybe (SubscriptExpr e, [], Nothing) res+      in+        maybe notSlice (\a -> (a, [], Nothing)) $ mkSlice e+      where+        mkSlice+          :: Raw Expr+          -> Maybe (Raw Subscript)+        mkSlice e = do+          c <- e ^? _Call+          case c ^? callFunction._Ident.identValue of+            Just "slice" ->+              pure $ case c ^.. callArguments.folded.folded of+                [PositionalArg _ x] ->+                  SubscriptSlice Nothing (MkColon []) (Just x) Nothing+                [PositionalArg _ x, PositionalArg _ y] ->+                  SubscriptSlice+                    (noneToMaybe x)+                    (MkColon [])+                    (noneToMaybe y)+                    Nothing+                [PositionalArg _ x, PositionalArg _ y, PositionalArg _ z] ->+                  SubscriptSlice+                    (noneToMaybe x)+                    (MkColon [])+                    (noneToMaybe y)+                    ((,) (MkColon []) . Just <$> noneToMaybe z)+                _ -> SubscriptExpr e+            _ -> Nothing++        noneToMaybe x = fromMaybe (Just x) $ Nothing <$ (x ^? _None)++        fromTupleItem+          :: Raw TupleItem+          -> Maybe (Raw Subscript)+        fromTupleItem (TupleItem _ a) = mkSlice a <|> pure (SubscriptExpr a)+        fromTupleItem _ = Nothing
+ src/Language/Python/Internal/Lexer.hs view
@@ -0,0 +1,763 @@+{-# language BangPatterns #-}+{-# language TypeApplications #-}+{-# language FunctionalDependencies, MultiParamTypeClasses #-}+{-# language GeneralizedNewtypeDeriving #-}+{-# language FlexibleContexts #-}+{-# language TypeFamilies #-}+{-# language OverloadedStrings #-}+{-# language LambdaCase #-}++{-|+Module      : Language.Python.Internal.Lexer+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Internal.Lexer+  ( tokenizeWithTabs+    -- * Source Information+  , SrcInfo(..), initialSrcInfo, withSrcInfo+    -- * Errors+  , AsLexicalError(..), unsafeFromLexicalError+  , AsTabError(..), AsIncorrectDedent(..), fromTabError, TabError(..)+    -- * Miscellaneous+  , tokenize+  , insertTabs+    -- * Megaparsec re-exports+  , Parsec.ParseError(..)+  )+where++import Control.Applicative ((<|>), many, optional)+import Control.Lens.Getter ((^.))+import Control.Lens.Iso (from)+import Control.Lens.Prism (Prism')+import Control.Lens.Review ((#))+import Control.Monad ((<=<), when, replicateM)+import Control.Monad.Except (throwError)+import Control.Monad.State (StateT, evalStateT, get, modify, put)+import Data.Bifunctor (first)+import Data.Digit.Binary (parseBinary)+import Data.Digit.Class.D0 (parse0)+import Data.Digit.Decimal (parseDecimal, parseDecimalNoZero)+import Data.Digit.Hexadecimal.MixedCase (parseHeXaDeCiMaL)+import Data.Digit.Octal (parseOctal)+import Data.FingerTree (FingerTree, Measured(..))+import Data.Foldable (asum)+import Data.Functor.Identity (Identity)+import Data.List.NonEmpty (NonEmpty(..), some1)+import Data.Monoid (Sum(..))+import Data.Set (Set)+import Data.Semigroup (Semigroup, (<>))+import Data.Semigroup.Foldable (foldMap1)+import Data.These (These(..))+import Data.Void (Void)+import GHC.Stack (HasCallStack)+import Text.Megaparsec (MonadParsec, ParseError, parse, unPos)+import Text.Megaparsec.Parsers+  ( ParsecT, CharParsing, LookAheadParsing, lookAhead, unParsecT, satisfy, text+  , char, manyTill, try+  , notFollowedBy, anyChar, digit, oneOf+  )++import qualified Data.FingerTree as FingerTree+import qualified Data.List.NonEmpty as NonEmpty+import qualified Data.Text as Text+import qualified Text.Megaparsec as Parsec++import Language.Python.Internal.Token (PyToken(..), pyTokenAnn)+import Language.Python.Syntax.Comment+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace++data SrcInfo+  = SrcInfo+  { _srcInfoName :: FilePath+  , _srcInfoLineStart :: !Int+  , _srcInfoLineEnd :: !Int+  , _srcInfoColStart :: !Int+  , _srcInfoColEnd :: !Int+  , _srcInfoOffsetStart :: !Int+  , _srcInfoOffsetEnd :: !Int+  }+  deriving (Eq, Show)++instance Semigroup SrcInfo where+  SrcInfo _ ls le cs ce os oe <> SrcInfo n' ls' le' cs' ce' os' oe' =+    SrcInfo n' (min ls ls') (max le le') (min cs cs') (max ce ce') (min os os') (max oe oe')++initialSrcInfo :: FilePath -> SrcInfo+initialSrcInfo fp = SrcInfo fp 0 0 0 0 0 0++{-# inline withSrcInfo #-}+withSrcInfo :: MonadParsec e s m => m (SrcInfo -> a) -> m a+withSrcInfo m =+  (\(Parsec.SourcePos name l c) o f (Parsec.SourcePos _ l' c') o' ->+     f $ SrcInfo name (unPos l) (unPos l') (unPos c) (unPos c') o o') <$>+  Parsec.getPosition <*>+  Parsec.getTokensProcessed <*>+  m <*>+  Parsec.getPosition <*>+  Parsec.getTokensProcessed++newline :: CharParsing m => m Newline+newline = LF <$ char '\n' <|> char '\r' *> (CRLF <$ char '\n' <|> pure CR)++parseNewline :: (CharParsing m, Monad m) => m (SrcInfo -> PyToken SrcInfo)+parseNewline = TkNewline <$> newline++parseComment :: (CharParsing m, Monad m) => m (SrcInfo -> PyToken SrcInfo)+parseComment =+  (\a b -> TkComment (MkComment b a)) <$ char '#' <*> many (satisfy (`notElem` ['\r', '\n']))++stringOrBytesPrefix+  :: CharParsing m+  => m (Either+          (Either RawStringPrefix StringPrefix)+          (Either RawBytesPrefix BytesPrefix))+stringOrBytesPrefix =+  (char 'r' *>+   (Right (Left Prefix_rb) <$ char 'b' <|>+    Right (Left Prefix_rB) <$ char 'B' <|>+    pure (Left $ Left Prefix_r))) <|>+  (char 'R' *>+   (Right (Left Prefix_Rb) <$ char 'b' <|>+    Right (Left Prefix_RB) <$ char 'B' <|>+    pure (Left $ Left Prefix_R))) <|>+  (char 'b' *>+   (Right (Left Prefix_br) <$ char 'r' <|>+    Right (Left Prefix_bR) <$ char 'R' <|>+    pure (Right $ Right Prefix_b))) <|>+  (char 'B' *>+   (Right (Left Prefix_Br) <$ char 'r' <|>+    Right (Left Prefix_BR) <$ char 'R' <|>+    pure (Right $ Right Prefix_B))) <|>+  (Left (Right Prefix_u) <$ char 'u') <|>+  (Left (Right Prefix_U) <$ char 'U')++rawStringChar :: CharParsing m => m [PyChar]+rawStringChar =+  (\a -> [Char_lit '\\', Char_lit a]) <$ char '\\' <*> anyChar <|>+  pure . Char_lit <$> anyChar++stringChar :: (CharParsing m, LookAheadParsing m) => m PyChar+stringChar =+  (try (char '\\' <* lookAhead (oneOf "\"'U\\abfntuvx01234567")) *>+   (escapeChar <|> unicodeChar <|> octChar <|> hexChar)) <|>+  other+  where+    other = Char_lit <$> anyChar+    escapeChar =+      asum @[]+      [ Char_esc_bslash <$ char '\\'+      , Char_esc_singlequote <$ char '\''+      , Char_esc_doublequote <$ char '"'+      , Char_esc_a <$ char 'a'+      , Char_esc_b <$ char 'b'+      , Char_esc_f <$ char 'f'+      , char 'n' *> (Char_newline <$ text "ewline" <|> pure Char_esc_n)+      , Char_esc_r <$ char 'r'+      , Char_esc_t <$ char 't'+      , Char_esc_v <$ char 'v'+      ]++    unicodeChar =+      char 'U' *>+      ((\[a, b, c, d, e, f, g, h] -> Char_uni32 a b c d e f g h) <$>+       replicateM 8 parseHeXaDeCiMaL)+      <|>+      char 'u' *>+      ((\[a, b, c, d] -> Char_uni16 a b c d) <$>+       replicateM 4 parseHeXaDeCiMaL)++    hexChar = Char_hex <$ char 'x' <*> parseHeXaDeCiMaL <*> parseHeXaDeCiMaL+    octChar =+      (\a b c ->+         maybe+           (Char_octal1 a)+           (\b' -> maybe (Char_octal2 a b') (Char_octal3 a b') c)+           b) <$>+      parseOctal <*>+      optional parseOctal <*>+      optional parseOctal++number :: (CharParsing m, Monad m) => m (a -> PyToken a)+number = do+  zero <- optional parse0+  case zero of+    Nothing -> do+      nn <- optional $ (:|) <$> parseDecimalNoZero <*> many parseDecimal+      case nn of+        Just n ->+          (\x j ann ->+             case x of+               Nothing ->+                 maybe (TkInt $ IntLiteralDec ann n) (TkImag . ImagLiteralInt ann n) j+               Just (Right e) ->+                 let+                   f = FloatLiteralWhole ann n e+                 in+                   maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j+               Just (Left (Left e)) ->+                 let+                   f = FloatLiteralFull ann n (Just (That e))+                 in+                   maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j+               Just (Left (Right (a, b))) ->+                 let+                   f = FloatLiteralFull ann n $+                     case (a, b) of+                       (Nothing, Nothing) -> Nothing+                       (Just x, Nothing) -> Just $ This x+                       (Nothing, Just x) -> Just $ That x+                       (Just x, Just y) -> Just $ These x y+                 in+                   maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j) <$>+          optional+            (Left <$ char '.' <*>+             (Left <$> floatExp <|>+              Right <$> ((,) <$> optional (some1 parseDecimal) <*> optional floatExp)) <|>+             Right <$> floatExp) <*>+          optional jJ+        Nothing ->+          (\a b j ann ->+             let+               f = FloatLiteralPoint ann a b+             in+               maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j) <$>+          -- try is necessary here to prevent the intercepting of dereference tokens+          try (char '.' *> some1 parseDecimal) <*>+          optional floatExp <*>+          optional jJ+    Just z ->+      (\xX a b -> TkInt (IntLiteralHex b xX a)) <$>+      (True <$ char 'X' <|> False <$ char 'x') <*>+      some1 parseHeXaDeCiMaL+      <|>+      (\bB a b -> TkInt (IntLiteralBin b bB a)) <$>+      (True <$ char 'B' <|> False <$ char 'b') <*>+      some1 parseBinary+      <|>+      (\oO a b -> TkInt (IntLiteralOct b oO a)) <$>+      (True <$ char 'O' <|> False <$ char 'o') <*>+      some1 parseOctal+      <|>+      (\n j a ->+         maybe (TkInt $ IntLiteralDec a (z :| n)) (TkImag . ImagLiteralInt a (z :| n)) j) <$>+      try (many parse0 <* notFollowedBy (char '.' <|> char 'e' <|> char 'E' <|> digit)) <*>+      optional jJ+      <|>+      (\n' a ann ->+         case a of+           Left (Left (b, c, j)) ->+             let+               f = FloatLiteralFull ann (z :| n') $+                 case (b, c) of+                   (Nothing, Nothing) -> Nothing+                   (Just x, Nothing) -> Just $ This x+                   (Nothing, Just x) -> Just $ That x+                   (Just x, Just y) -> Just $ These x y+             in+               maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j+           Left (Right (x, j)) ->+             let+               f = FloatLiteralWhole ann (z :| n') x+             in+               maybe (TkFloat f) (TkImag . ImagLiteralFloat ann f) j+           Right j -> TkImag $ ImagLiteralInt ann (z :| n') j) <$>+      many parseDecimal <*>+      (Left <$>+       (Left <$>+        ((,,) <$ char '.' <*>+         optional (some1 parseDecimal) <*>+         optional floatExp <*>+         optional jJ) <|>+        Right <$>+        ((,) <$> floatExp <*> optional jJ)) <|>+      Right <$> jJ)+  where+    jJ = False <$ char 'j' <|> True <$ char 'J'+    floatExp =+      FloatExponent <$>+      (EE <$ char 'E' <|> Ee <$ char 'e') <*>+      optional (Pos <$ char '+' <|> Neg <$ char '-') <*>+      some1 parseDecimal++{-# inline parseToken #-}+parseToken+  :: (Monad m, CharParsing m, LookAheadParsing m, MonadParsec e s m)+  => m (PyToken SrcInfo)+parseToken =+  withSrcInfo $+  try+    (asum+     [ TkIf <$ text "if"+     , TkElse <$ text "else"+     , TkElif <$ text "elif"+     , TkWhile <$ text "while"+     , TkAssert <$ text "assert"+     , TkDef <$ text "def"+     , TkReturn <$ text "return"+     , TkPass <$ text "pass"+     , TkBreak <$ text "break"+     , TkContinue <$ text "continue"+     , TkTrue <$ text "True"+     , TkFalse <$ text "False"+     , TkNone <$ text "None"+     , TkOr <$ text "or"+     , TkAnd <$ text "and"+     , TkIs <$ text "is"+     , TkNot <$ text "not"+     , TkGlobal <$ text "global"+     , TkNonlocal <$ text "nonlocal"+     , TkDel <$ text "del"+     , TkLambda <$ text "lambda"+     , TkImport <$ text "import"+     , TkFrom <$ text "from"+     , TkAs <$ text "as"+     , TkRaise <$ text "raise"+     , TkTry <$ text "try"+     , TkExcept <$ text "except"+     , TkFinally <$ text "finally"+     , TkClass <$ text "class"+     , TkWith <$ text "with"+     , TkFor <$ text "for"+     , TkIn <$ text "in"+     , TkYield <$ text "yield"+     ] <* notFollowedBy (satisfy isIdentifierChar))++    <|>++    asum+    [ number+    , TkRightArrow <$ text "->"+    , TkEllipsis <$ text "..."+    , TkSpace <$ char ' '+    , TkTab <$ char '\t'+    , TkLeftBracket <$ char '['+    , TkRightBracket <$ char ']'+    , TkLeftParen <$ char '('+    , TkRightParen <$ char ')'+    , TkLeftBrace <$ char '{'+    , TkRightBrace <$ char '}'+    , char '<' *>+      (TkLte <$ char '=' <|>+       char '<' *> (TkShiftLeftEq <$ char '=' <|> pure TkShiftLeft) <|>+       pure TkLt)+    , char '=' *> (TkDoubleEq <$ char '=' <|> pure TkEq)+    , char '>' *>+      (TkGte <$ char '=' <|>+       char '>' *> (TkShiftRightEq <$ char '=' <|> pure TkShiftRight) <|>+       pure TkGt)+    , char '*' *>+      (char '*' *> (TkDoubleStarEq <$ char '=' <|> pure TkDoubleStar) <|>+       TkStarEq <$ char '=' <|>+       pure TkStar)+    , char '/' *>+      (char '/' *> (TkDoubleSlashEq <$ char '=' <|> pure TkDoubleSlash) <|>+       TkSlashEq <$ char '=' <|>+       pure TkSlash)+    , TkBangEq <$ text "!="+    , char '^' *> (TkCaretEq <$ char '=' <|> pure TkCaret)+    , char '|' *> (TkPipeEq <$ char '=' <|> pure TkPipe)+    , char '&' *> (TkAmpersandEq <$ char '=' <|> pure TkAmpersand)+    , char '@' *> (TkAtEq <$ char '=' <|> pure TkAt)+    , char '+' *> (TkPlusEq <$ char '=' <|> pure TkPlus)+    , char '-' *> (TkMinusEq <$ char '=' <|> pure TkMinus)+    , char '%' *> (TkPercentEq <$ char '=' <|> pure TkPercent)+    , TkTilde <$ char '~'+    , TkContinued <$ char '\\' <*> newline+    , TkColon <$ char ':'+    , TkSemicolon <$ char ';'+    , parseComment+    , parseNewline+    , TkComma <$ char ','+    , TkDot <$ char '.'+    , do+        sp <- try $ optional stringOrBytesPrefix <* char '"'+        case sp of+          Nothing ->+            TkString Nothing LongString DoubleQuote <$+            text "\"\"" <*>+            manyTill stringChar (text "\"\"\"")+            <|>+            TkString Nothing ShortString DoubleQuote <$> manyTill stringChar (char '"')+          Just (Left (Left prefix)) ->+            TkRawString prefix LongString DoubleQuote . concat <$+            text "\"\"" <*>+            manyTill rawStringChar (text "\"\"\"")+            <|>+            TkRawString prefix ShortString DoubleQuote . concat <$>+            manyTill rawStringChar (char '"')+          Just (Left (Right prefix)) ->+            TkString (Just prefix) LongString DoubleQuote <$+            text "\"\"" <*>+            manyTill stringChar (text "\"\"\"")+            <|>+            TkString (Just prefix) ShortString DoubleQuote <$> manyTill stringChar (char '"')+          Just (Right (Left prefix)) ->+            TkRawBytes prefix LongString DoubleQuote . concat <$+            text "\"\"" <*>+            manyTill rawStringChar (text "\"\"\"")+            <|>+            TkRawBytes prefix ShortString DoubleQuote . concat <$>+            manyTill rawStringChar (char '"')+          Just (Right (Right prefix)) ->+            TkBytes prefix LongString DoubleQuote <$+            text "\"\"" <*>+            manyTill stringChar (text "\"\"\"")+            <|>+            TkBytes prefix ShortString DoubleQuote <$> manyTill stringChar (char '"')+    , do+        sp <- try $ optional stringOrBytesPrefix <* char '\''+        case sp of+          Nothing ->+            TkString Nothing LongString SingleQuote <$+            text "''" <*>+            manyTill stringChar (text "'''")+            <|>+            TkString Nothing ShortString SingleQuote <$> manyTill stringChar (char '\'')+          Just (Left (Left prefix)) ->+            TkRawString prefix LongString SingleQuote . concat <$+            text "''" <*>+            manyTill rawStringChar (text "'''")+            <|>+            TkRawString prefix ShortString SingleQuote . concat <$>+            manyTill rawStringChar (char '\'')+          Just (Left (Right prefix)) ->+            TkString (Just prefix) LongString SingleQuote <$+            text "''" <*>+            manyTill stringChar (text "'''")+            <|>+            TkString (Just prefix) ShortString SingleQuote <$> manyTill stringChar (char '\'')+          Just (Right (Left prefix)) ->+            TkRawBytes prefix LongString SingleQuote . concat <$+            text "''" <*>+            manyTill rawStringChar (text "'''")+            <|>+            TkRawBytes prefix ShortString SingleQuote . concat <$>+            manyTill rawStringChar (char '\'')+          Just (Right (Right prefix)) ->+            TkBytes prefix LongString SingleQuote <$+            text "''" <*>+            manyTill stringChar (text "'''")+            <|>+            TkBytes prefix ShortString SingleQuote <$> manyTill stringChar (char '\'')+    , fmap TkIdent $+      (:) <$>+      satisfy isIdentifierStart <*>+      many (satisfy isIdentifierChar)+    ]++class AsLexicalError s t | s -> t where+  _LexicalError+    :: Prism'+         s+         ( NonEmpty Parsec.SourcePos+         , Maybe (Parsec.ErrorItem t)+         , Set (Parsec.ErrorItem t)+         )++-- | Convert a concrete 'ParseError' to a value that has an instance of 'AsLexicalError'+--+-- This function is partial, because our parser will never use 'Parsec.FancyError'+unsafeFromLexicalError+  :: ( HasCallStack+     , AsLexicalError s t+     )+  => ParseError t Void+  -> s+unsafeFromLexicalError (Parsec.TrivialError a b c) = _LexicalError # (a, b, c)+unsafeFromLexicalError Parsec.FancyError{} = error "'fancy error' used in lexer"++{-# noinline tokenize #-}+-- | Convert some input to a sequence of tokens. Indent and dedent tokens are not added+-- (see 'insertTabs')+tokenize+  :: AsLexicalError e Char+  => FilePath -- ^ File name+  -> Text.Text -- ^ Input to tokenize+  -> Either e [PyToken SrcInfo]+tokenize fp = first unsafeFromLexicalError . parse (unParsecT tokens) fp+  where+    tokens :: ParsecT Void Text.Text Identity [PyToken SrcInfo]+    tokens = many parseToken <* Parsec.eof++data LogicalLine a+  = LogicalLine+      a -- annotation+      ([PyToken a], Indent) -- spaces+      [PyToken a] -- line+      (Maybe (PyToken a)) -- end+  | BlankLine+      [PyToken a] -- line+      (Maybe (PyToken a)) -- end+  deriving (Eq, Show)++logicalLineToTokens :: LogicalLine a -> [PyToken a]+logicalLineToTokens (LogicalLine _ _ ts m) = ts <> maybe [] pure m+logicalLineToTokens (BlankLine ts m) = ts <> maybe [] pure m++spaceToken :: PyToken a -> Maybe Whitespace+spaceToken TkSpace{} = Just Space+spaceToken TkTab{} = Just Tab+spaceToken (TkContinued nl _) = Just $ Continued nl []+spaceToken _ = Nothing++collapseContinue :: [(PyToken a, Whitespace)] -> [([PyToken a], Whitespace)]+collapseContinue [] = []+collapseContinue ((tk@TkSpace{}, Space) : xs) =+  ([tk], Space) : collapseContinue xs+collapseContinue ((tk@TkTab{}, Tab) : xs) =+  ([tk], Tab) : collapseContinue xs+collapseContinue ((tk@TkNewline{}, Newline nl) : xs) =+  ([tk], Newline nl) : collapseContinue xs+collapseContinue ((tk@TkContinued{}, Continued nl ws) : xs) =+  let+    xs' = collapseContinue xs+  in+    [(tk : (xs' >>= fst), Continued nl $ ws <> fmap snd xs')]+collapseContinue _ = error "invalid token/whitespace pair in collapseContinue"++spanMaybe :: (a -> Maybe b) -> [a] -> ([b], [a])+spanMaybe f as =+  case as of+    [] -> ([], [])+    x : xs ->+      case f x of+        Nothing -> ([], as)+        Just b -> first (b :) $ spanMaybe f xs++-- | Acts like break, but encodes the "insignificant whitespace" rule for parens, braces+-- and brackets+breakOnNewline :: [PyToken a] -> ([PyToken a], Maybe (PyToken a, [PyToken a]))+breakOnNewline = go 0+  where+    go :: Int -> [PyToken a] -> ([PyToken a], Maybe (PyToken a, [PyToken a]))+    go _ [] = ([], Nothing)+    go !careWhen0 (tk : tks) =+      case tk of+        TkLeftParen{} -> first (tk :) $ go (careWhen0 + 1) tks+        TkLeftBracket{} -> first (tk :) $ go (careWhen0 + 1) tks+        TkLeftBrace{} -> first (tk :) $ go (careWhen0 + 1) tks+        TkRightParen{} -> first (tk :) $ go (max 0 $ careWhen0 - 1) tks+        TkRightBracket{} -> first (tk :) $ go (max 0 $ careWhen0 - 1) tks+        TkRightBrace{} -> first (tk :) $ go (max 0 $ careWhen0 - 1) tks+        TkNewline{}+          | careWhen0 == 0 -> ([], Just (tk, tks))+          | otherwise -> first (tk :) $ go careWhen0 tks+        _ -> first (tk :) $ go careWhen0 tks++logicalLines :: [PyToken a] -> [LogicalLine a]+logicalLines [] = []+logicalLines tks =+  let+    (spaces, rest) = spanMaybe (\a -> (,) a <$> spaceToken a) tks+    (line, rest') = breakOnNewline rest+    spaces' = collapseContinue spaces+  in+    (if+       not (any (\case; Continued{} -> True; _ -> False) $ snd <$> spaces) &&+       all isBlankToken line+     then+       BlankLine (fmap fst spaces <> line) (fst <$> rest')+     else+       LogicalLine+         (case tks of+           [] -> error "couldn't generate annotation for logical line"+           tk : _ -> pyTokenAnn tk)+         (spaces' >>= fst, fmap snd spaces' ^. from indentWhitespaces)+         line+         (fst <$> rest')) :+    logicalLines (maybe [] snd rest')++data IndentedLine a+  = Indent Int Indent a+  | Level (NonEmpty Whitespace) a+  | Dedent a+  | IndentedLine (LogicalLine a)+  deriving (Eq, Show)++isBlankToken :: PyToken a -> Bool+isBlankToken TkSpace{} = True+isBlankToken TkTab{} = True+isBlankToken TkComment{} = True+isBlankToken TkNewline{} = True+isBlankToken _ = False++data TabError a+  -- | Tabs and spaces were used inconsistently+  = TabError a+  -- | The dedent at the end of a block doesn't match and preceding indents+  --+  -- e.g.+  --+  -- @+  -- def a():+  --     if b:+  --         pass+  --     else:+  --         pass+  --   pass+  -- @+  --+  -- The final line will cause an 'IncorrectDedent' error+  | IncorrectDedent a+  deriving (Eq, Show)++class AsTabError s a | s -> a where+  _TabError :: Prism' s a++class AsIncorrectDedent s a | s -> a where+  _IncorrectDedent :: Prism' s a++-- | Convert a concrete 'TabError' to a value that has an instance of 'AsTabError'+fromTabError+  :: ( AsTabError s a+     , AsIncorrectDedent s a+     )+  => TabError a -> s+fromTabError (TabError a) = _TabError # a+fromTabError (IncorrectDedent a) = _IncorrectDedent # a++indentation :: Semigroup a => a -> [LogicalLine a] -> Either (TabError a) [IndentedLine a]+indentation ann lls =+  flip evalStateT (pure (ann, mempty)) $+  (<>) <$> (concat <$> traverse go lls) <*> finalDedents+  where+    finalDedents :: StateT (NonEmpty (a, Indent)) (Either (TabError a)) [IndentedLine a]+    finalDedents = do+      (ann, _) :| is <- get+      case is of+        [] -> pure []+        i' : is' -> do+          put $ i' :| is'+          (Dedent ann :) <$> finalDedents++    dedents+      :: a+      -> Int+      -> StateT (NonEmpty (a, Indent)) (Either (TabError a)) [IndentedLine a]+    dedents ann n = do+      is <- get+      let (popped, remainder) = NonEmpty.span ((> n) . indentLevel . snd) is+      when (n `notElem` fmap (indentLevel . snd) (NonEmpty.toList is)) .+        throwError $ IncorrectDedent ann+      put $ case remainder of+        [] -> error "I don't know whether this can happen"+        x : xs -> x :| xs+      pure $ replicate (length popped) (Dedent ann)++    go+      :: Semigroup a+      => LogicalLine a+      -> StateT (NonEmpty (a, Indent)) (Either (TabError a)) [IndentedLine a]+    go ll@BlankLine{} = pure [IndentedLine ll]+    go ll@(LogicalLine ann (spTks, spcs) _ _) = do+      (_, i) :| _ <- get+      let+        et8 = absoluteIndentLevel 8 spcs+        et1 = absoluteIndentLevel 1 spcs+        et8i = absoluteIndentLevel 8 i+        et1i = absoluteIndentLevel 1 i+      when+        (not (et8 < et8i && et1 < et1i) &&+          not (et8 > et8i && et1 > et1i) &&+          not (et8 == et8i && et1 == et1i))+        (throwError $ TabError ann)+      let+        ilSpcs = indentLevel spcs+        ili = indentLevel i+        levelIndent =+          case (spTks, spcs ^. indentWhitespaces) of+            ([], []) -> []+            (x:xs, y:ys) -> [ Level (y:|ys) (foldMap1 pyTokenAnn $ x:|xs) ]+            _ -> error "impossible"+      case compare ilSpcs ili of+        LT -> (<> (levelIndent <> [IndentedLine ll])) <$> dedents ann ilSpcs+        EQ ->+          pure $ levelIndent <> [ IndentedLine ll ]+        GT -> do+          modify $ NonEmpty.cons (ann, spcs)+          pure [Indent (ilSpcs - ili) spcs ann, IndentedLine ll]++newtype Summed a = Summed a+  deriving (Eq, Show, Ord, Num)++instance Num a => Measured (Sum a) (Summed a) where+  measure (Summed a) = Sum a++-- | Given a list of indentation jumps (first to last) and some whitespace,+-- divide the whitespace up into "blocks" which correspond to each jump+splitIndents :: FingerTree (Sum Int) (Summed Int) -> Indent -> [Indent]+splitIndents ns ws = go ns ws []+  where+    go :: FingerTree (Sum Int) (Summed Int) -> Indent -> [Indent] -> [Indent]+    go ns ws =+      case FingerTree.viewr ns of+        FingerTree.EmptyR -> (ws :)+        ns' FingerTree.:> n+          | FingerTree.null ns' -> (ws :)+          | otherwise ->+              let+                (befores, afters) =+                  FingerTree.split ((> getSum (measure ns')) . getIndentLevel) $ unIndent ws+              in+                if FingerTree.null afters+                then error $ "could not carve out " <> show n <> " from " <> show ws+                else go ns' (MkIndent befores) . (MkIndent afters :)++chunked :: [IndentedLine a] -> [PyToken a]+chunked = go FingerTree.empty+  where+    go+      :: FingerTree (Sum Int) (Summed Int)+      -> [IndentedLine a]+      -> [PyToken a]+    go _ [] = []+    go leaps (Indent n i a : is) =+      let+        leaps' = leaps FingerTree.|> Summed n+      in+        TkIndent a (Indents (splitIndents leaps' i) a) : go leaps' is+    go leaps (Dedent a : is) =+      case FingerTree.viewr leaps of+        FingerTree.EmptyR -> error "impossible"+        leaps' FingerTree.:> _ -> TkDedent a : go leaps' is+    go leaps (IndentedLine ll : is) = logicalLineToTokens ll <> go leaps is+    go leaps (Level i a : is) =+      TkLevel a (Indents (splitIndents leaps $ NonEmpty.toList i ^. from indentWhitespaces) a) : go leaps is++-- | Insert indent and dedent tokens+--+-- https://docs.python.org/3.5/reference/lexical_analysis.html#indentation+insertTabs+  :: ( Semigroup a+     , AsTabError s a+     , AsIncorrectDedent s a+     )+  => a -- ^ Initial source annotation+  -> [PyToken a] -- ^ Token stream+  -> Either s [PyToken a]+insertTabs a =+  first fromTabError .+  fmap chunked .+  indentation a .+  logicalLines++-- | Tokenize an input file, inserting indent\/level\/dedent tokens in appropriate+-- positions according to the block structure.+tokenizeWithTabs+  :: ( AsLexicalError s Char+     , AsTabError s SrcInfo+     , AsIncorrectDedent s SrcInfo+     )+  => FilePath -- ^ File name+  -> Text.Text -- ^ Input to tokenize+  -> Either s [PyToken SrcInfo]+tokenizeWithTabs fp = insertTabs (initialSrcInfo fp) <=< tokenize fp
+ src/Language/Python/Internal/Parse.hs view
@@ -0,0 +1,1575 @@+{-# language DataKinds #-}+{-# language FlexibleContexts #-}+{-# language LambdaCase #-}+{-# language RankNTypes #-}+{-# language FunctionalDependencies, MultiParamTypeClasses #-}+{-# language TypeFamilies #-}++{-|+Module      : Language.Python.Internal.Parse+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Internal.Parse+  ( Parser+  , runParser+    -- * Stream type+  , PyTokens(..)+    -- * Errors+  , AsParseError(..)+  , unsafeFromParseError+    -- * Parsers+  , token+    -- ** Symbols+  , at+  , colon+  , comma+  , dot+  , doubleStar+  , equals+  , rightParen+  , semicolon+  , star+    -- ** Atomic forms+  , identifier+  , bool+  , none+  , ellipsis+  , integer+  , float+  , imag+  , stringOrBytes+    -- ** Compound forms+  , arg+  , binOp+  , commaSep+  , commaSep1+  , commaSep1'+  , commaSepRest+  , compIf+  , compFor+  , compoundStatement+  , decorator+  , decoratorValue+  , decorators+  , expr+  , exprList+  , exprListComp+  , exprNoCond+  , exprComp+  , exprOrStarList+  , lambda+  , lambdaNoCond+  , module_+  , orExpr+  , orExprList+  , orTest+  , smallStatement+  , someParams+  , simpleStatement+  , starExpr+  , statement+  , suite+  , tpPositional+  , tpStar+  , tpDoubleStar+  , tyAnn+  , typedParams+  , untypedParams+  , upPositional+  , upStar+  , upDoubleStar+  , yieldExpr+    -- ** Formatting+  , anySpace+  , space+  , eol+  , continued+  , newline+  , indent+  , dedent+  , level+  , blank+  , comment+    -- ** Miscellaneous combinators+  , sepBy1'+  )+where++import Control.Applicative (Alternative, (<|>), optional, many, some)+import Control.Lens.Cons (snoc)+import Control.Lens.Getter ((^.), view)+import Control.Lens.Prism (Prism')+import Control.Lens.Review ((#))+import Control.Monad (void)+import Data.Bifunctor (first, second)+import Data.Coerce (coerce)+import Data.Function ((&))+import Data.List (foldl')+import Data.List.NonEmpty (NonEmpty, some1)+import Data.Proxy (Proxy(..))+import Data.Set (Set)+import Data.Void (Void)+import GHC.Stack (HasCallStack)+import Text.Megaparsec+  ( (<?>), MonadParsec, Parsec, Stream(..), SourcePos(..), eof, try, lookAhead+  , notFollowedBy+  )+import Text.Megaparsec.Char (satisfy)+++import qualified Data.List.NonEmpty as NonEmpty+import qualified Text.Megaparsec as Megaparsec++import Language.Python.Internal.Lexer (SrcInfo(..), withSrcInfo)+import Language.Python.Internal.Syntax.IR+import Language.Python.Internal.Token+import Language.Python.Syntax.AugAssign+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Comment+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Import+import Language.Python.Syntax.ModuleNames+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Operator.Binary+import Language.Python.Syntax.Operator.Unary+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace++newtype PyTokens = PyTokens { unPyTokens :: [PyToken SrcInfo] }+  deriving (Eq, Ord)++instance Stream PyTokens where+  type Token PyTokens = PyToken SrcInfo+  type Tokens PyTokens = PyTokens+  tokenToChunk Proxy = PyTokens . pure+  tokensToChunk Proxy = PyTokens+  chunkToTokens Proxy = unPyTokens+  chunkLength Proxy = length . unPyTokens+  chunkEmpty Proxy = null . unPyTokens+  positionAt1 Proxy _ tk =+    let+      ann = pyTokenAnn tk+    in+      SourcePos+        (_srcInfoName ann)+        (Megaparsec.mkPos $ _srcInfoLineStart ann)+        (Megaparsec.mkPos $ _srcInfoColStart ann)+  positionAtN Proxy spos (PyTokens tks) =+    case tks of+      [] -> spos+      _ ->+        let+          ann = pyTokenAnn $ last tks+        in+          SourcePos+            (_srcInfoName ann)+            (Megaparsec.mkPos $ _srcInfoLineStart ann)+            (Megaparsec.mkPos $ _srcInfoColStart ann)+  advance1 Proxy _ _ tk =+    let+      ann = pyTokenAnn tk+    in+      SourcePos+        (_srcInfoName ann)+        (Megaparsec.mkPos $ _srcInfoLineEnd ann)+        (Megaparsec.mkPos $ _srcInfoColEnd ann)+  advanceN Proxy _ spos (PyTokens tks) =+    case tks of+      [] -> spos+      _ ->+        let+          ann = pyTokenAnn $ last tks+        in+          SourcePos+            (_srcInfoName ann)+            (Megaparsec.mkPos $ _srcInfoLineEnd ann)+            (Megaparsec.mkPos $ _srcInfoColEnd ann)++  take1_ (PyTokens p) =+    case p of+      [] -> Nothing+      t:ts -> Just (t, PyTokens ts)++  takeN_ n (PyTokens s)+    | n <= 0    = Just (PyTokens [], PyTokens s)+    | null s    = Nothing+    | otherwise = Just (coerce (splitAt n s))++  takeWhile_ f = coerce (span f)++class AsParseError s t | s -> t where+  _ParseError+    :: Prism'+         s+         ( NonEmpty SourcePos+         , Maybe (Megaparsec.ErrorItem t)+         , Set (Megaparsec.ErrorItem t)+         )++-- | Convert a concrete 'Megaparsec.ParseError' to a value that has an instance of 'AsParseError'+--+-- This function is partial because our parser will never use 'Megaparsec.FancyError'+unsafeFromParseError+  :: (HasCallStack, AsParseError s t)+  => Megaparsec.ParseError t e+  -> s+unsafeFromParseError Megaparsec.FancyError{} = error "there are none of these"+unsafeFromParseError (Megaparsec.TrivialError pos a b) = _ParseError # (pos, a, b)++type Parser = Parsec Void PyTokens++-- | Run a parser on some input+{-# inline runParser #-}+runParser+  :: AsParseError e (PyToken SrcInfo)+  => FilePath -- ^ File name+  -> Parser a -- ^ Parser+  -> [PyToken SrcInfo] -- ^ Input to parse+  -> Either e a+runParser file p input =+  first unsafeFromParseError $ Megaparsec.parse p file (PyTokens input)++eol :: MonadParsec e PyTokens m => m Newline+eol =+  (\(TkNewline nl _) -> nl) <$>+  satisfy (\case; TkNewline{} -> True; _ -> False) <?> "newline"++dedent :: MonadParsec e PyTokens m => m ()+dedent = () <$ satisfy (\case; TkDedent{} -> True; _ -> False) <?> "dedent"++space :: MonadParsec e PyTokens m => m Whitespace+space =+  Space <$ satisfy (\case; TkSpace{} -> True; _ -> False) <|>+  Tab <$ satisfy (\case; TkTab{} -> True; _ -> False) <|>+  continued++continued :: MonadParsec e PyTokens m => m Whitespace+continued =+  (\(TkContinued nl _) -> Continued nl) <$>+  satisfy (\case; TkContinued{} -> True; _ -> False) <*>+  many space++newline :: MonadParsec e PyTokens m => m Newline+newline = (\(TkNewline nl _) -> nl) <$> satisfy (\case; TkNewline{} -> True; _ -> False)++anySpace :: MonadParsec e PyTokens m => m Whitespace+anySpace =+  Space <$ satisfy (\case; TkSpace{} -> True; _ -> False) <|>+  Tab <$ satisfy (\case; TkTab{} -> True; _ -> False) <|>+  continued <|>+  Newline <$> newline <|>+  Comment . void <$> comment++token+  :: MonadParsec e PyTokens m+  => m Whitespace+  -> (PyToken SrcInfo -> Bool)+  -> String+  -> m (PyToken SrcInfo, [Whitespace])+token ws f label = (,) <$> satisfy f <*> many ws <?> label++identifier :: MonadParsec e PyTokens m => m Whitespace -> m (Ident '[] SrcInfo)+identifier ws =+  (\(TkIdent n ann) -> MkIdent ann n) <$>+  satisfy (\case; TkIdent{} -> True; _ -> False) <*>+  many ws++bool :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+bool ws =+  (\(tk, s) ->+     Bool+       (pyTokenAnn tk)+       (case tk of+          TkTrue{} -> True+          TkFalse{} -> False+          _ -> error "impossible")+       s) <$>+  (token ws (\case; TkTrue{} -> True; _ -> False) "True" <|>+   token ws (\case; TkFalse{} -> True; _ -> False) "False")++none :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+none ws =+  (\(tk, s) -> None (pyTokenAnn tk) s) <$>+  token ws (\case; TkNone{} -> True; _ -> False) "None"++ellipsis :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+ellipsis ws =+  (\(tk, s) -> Ellipsis (pyTokenAnn tk) s) <$>+  token ws (\case; TkEllipsis{} -> True; _ -> False) "..."++integer :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+integer ws =+  (\(TkInt n) -> Int (_intLiteralAnn n) n) <$>+  satisfy (\case; TkInt{} -> True; _ -> False) <*>+  many ws++float :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+float ws =+  (\(TkFloat n) -> Float (_floatLiteralAnn n) n) <$>+  satisfy (\case; TkFloat{} -> True; _ -> False) <*>+  many ws++imag :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+imag ws =+  (\(TkImag n) -> Imag (_imagLiteralAnn n) n) <$>+  satisfy (\case; TkImag{} -> True; _ -> False) <*>+  many ws++stringOrBytes :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+stringOrBytes ws =+  fmap (\vs -> String (_stringLiteralAnn $ NonEmpty.head vs) vs) . some1 $+  (\case+     TkString sp qt st val ann -> StringLiteral ann sp qt st val+     TkBytes sp qt st val ann -> BytesLiteral ann sp qt st val+     TkRawString sp st qt val ann -> RawStringLiteral ann sp st qt val+     TkRawBytes sp st qt val ann -> RawBytesLiteral ann sp st qt val+     _ -> error "impossible") <$>+  satisfy+    (\case+        TkString{} -> True+        TkBytes{} -> True+        TkRawString{} -> True+        TkRawBytes{} -> True+        _ -> False) <*>+  many ws++comment :: MonadParsec e PyTokens m => m (Comment SrcInfo)+comment =+  (\(TkComment c) -> c) <$>+  satisfy (\case; TkComment{} -> True; _ -> False) <?> "comment"++indent :: MonadParsec e PyTokens m => m (Indents SrcInfo)+indent =+  (\(TkIndent _ i) -> i) <$> satisfy (\case; TkIndent{} -> True; _ -> False) <?> "indent"++level :: MonadParsec s PyTokens m => m (Indents SrcInfo)+level =+  (\(TkLevel _ i) -> i) <$> satisfy (\case; TkLevel{} -> True; _ -> False) <?> "level indentation"++comma :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Comma)+comma ws = second MkComma <$> token ws (\case; TkComma{} -> True; _ -> False) ","++dot :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Dot)+dot ws = second MkDot <$> token ws (\case; TkDot{} -> True; _ -> False) "."++at :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, At)+at ws = second MkAt <$> token ws (\case; TkAt{} -> True; _ -> False) "@"++colon :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Colon)+colon ws = second MkColon <$> token ws (\case; TkColon{} -> True; _ -> False) ":"++equals :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Equals)+equals ws = second MkEquals <$> token ws (\case; TkEq{} -> True; _ -> False) "="++semicolon :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, Semicolon SrcInfo)+semicolon ws =+  (\(a, b) -> (a, MkSemicolon (pyTokenAnn a) b)) <$>+  token ws (\case; TkSemicolon{} -> True; _ -> False) ";"++exprList :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+exprList ws =+  (\e -> maybe e (uncurry $ Tuple (e ^. exprAnn) e)) <$>+  expr ws <*>+  optional+    ((,) <$>+     (snd <$> comma ws) <*>+     optional (commaSep1' ws $ expr ws))++exprOrStarList :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+exprOrStarList ws =+  (\e -> maybe e (uncurry $ Tuple (e ^. exprAnn) e)) <$>+  (expr ws <|> starExpr ws) <*>+  optional+    ((,) <$>+     (snd <$> comma ws) <*>+     optional (commaSep1' ws $ expr ws <|> starExpr ws))++compIf :: MonadParsec e PyTokens m => m (CompIf SrcInfo)+compIf =+  (\(tk, s) -> CompIf (pyTokenAnn tk) s) <$>+  token anySpace (\case; TkIf{} -> True; _ -> False) "if" <*>+  exprNoCond anySpace++compFor :: MonadParsec e PyTokens m => m (CompFor SrcInfo)+compFor =+  (\(tk, s) -> CompFor (pyTokenAnn tk) s) <$>+  token anySpace (\case; TkFor{} -> True; _ -> False) "for" <*>+  orExprList anySpace <*>+  (snd <$> token anySpace (\case; TkIn{} -> True; _ -> False) "in") <*>+  orTest anySpace++commaSepRest :: MonadParsec e PyTokens m => m b -> m ([(Comma, b)], Maybe Comma)+commaSepRest x = do+  c <- optional $ snd <$> comma anySpace+  case c of+    Nothing -> pure ([], Nothing)+    Just c' -> do+      e <- optional x+      case e of+        Nothing -> pure ([], Just c')+        Just e' -> first ((c', e') :) <$> commaSepRest x++exprComp :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+exprComp ws =+  (\ex a ->+     case a of+       Nothing -> ex+       Just (cf, rest) ->+         Generator (ex ^. exprAnn) $+         Comprehension (ex ^. exprAnn) ex cf rest) <$>+  expr ws <*>+  optional ((,) <$> compFor <*> many (Left <$> compFor <|> Right <$> compIf))++star :: MonadParsec e PyTokens m => m Whitespace -> m (PyToken SrcInfo, [Whitespace])+star sp = token sp (\case; TkStar{} -> True; _ -> False) "*"++starExpr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+starExpr ws =+  (\(tk, sp) -> StarExpr (pyTokenAnn tk) sp) <$>+  star ws <*>+  orExpr ws++exprListComp :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+exprListComp ws =+  (\e a ->+     case a of+       Left (cf, cfs) ->+         let+           ann = e ^. exprAnn+         in+           Generator ann $ Comprehension ann e cf cfs+       Right (Just (c, cs)) -> Tuple (e ^. exprAnn) e c cs+       Right Nothing -> e) <$>+  (expr ws <|> starExpr ws) <*>+  (Left <$>+   ((,) <$>+    compFor <*>+    many (Left <$> compFor <|> Right <$> compIf)) <|>+   Right <$>+   optional+     ((,) <$>+      (snd <$> comma ws) <*>+      optional (commaSep1' ws $ expr ws <|> starExpr ws)))++orExprList :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+orExprList ws =+  (\e -> maybe e (uncurry $ Tuple (e ^. exprAnn) e)) <$>+  (orExpr ws <|> starExpr ws) <*>+  optional+    ((,) <$>+     (snd <$> comma ws) <*>+     optional (commaSep1' ws $ orExpr ws <|> starExpr ws))++binOp :: MonadParsec e PyTokens m => m (BinOp SrcInfo) -> m (Expr SrcInfo) -> m (Expr SrcInfo)+binOp op tm =+  (\t ts ->+      case ts of+        [] -> t+        _ -> foldl (\tm (o, val) -> BinOp (tm ^. exprAnn) tm o val) t ts) <$>+  tm <*>+  many ((,) <$> op <*> tm)++orTest :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+orTest ws = binOp orOp andTest+  where+    orOp =+      (\(tk, ws) -> BoolOr (pyTokenAnn tk) ws) <$>+      token ws (\case; TkOr{} -> True; _ -> False) "or"++    andOp =+      (\(tk, ws) -> BoolAnd (pyTokenAnn tk) ws) <$>+      token ws (\case; TkAnd{} -> True; _ -> False) "and"+    andTest = binOp andOp notTest++    notTest =+      (\(tk, s) -> Not (pyTokenAnn tk) s) <$>+      token ws (\case; TkNot{} -> True; _ -> False) "not" <*> notTest <|>+      comparison++    compOp =+      (\(tk, ws) -> maybe (Is (pyTokenAnn tk) ws) (IsNot (pyTokenAnn tk) ws)) <$>+      token ws (\case; TkIs{} -> True; _ -> False) "is" <*>+      optional (snd <$> token ws (\case; TkNot{} -> True; _ -> False) "not")++      <|>++      (\(tk, ws) -> NotIn (pyTokenAnn tk) ws) <$>+      token ws (\case; TkNot{} -> True; _ -> False) "not" <*>+      (snd <$> token ws (\case; TkIn{} -> True; _ -> False) "in")++      <|>++      (\(tk, ws) -> In (pyTokenAnn tk) ws) <$>+      token ws (\case; TkIn{} -> True; _ -> False) "in"++      <|>++      (\(tk, ws) -> Eq (pyTokenAnn tk) ws) <$>+      token ws (\case; TkDoubleEq{} -> True; _ -> False) "=="++      <|>++      (\(tk, ws) -> Lt (pyTokenAnn tk) ws) <$>+      token ws (\case; TkLt{} -> True; _ -> False) "<"++      <|>++      (\(tk, ws) -> LtEq (pyTokenAnn tk) ws) <$>+      token ws (\case; TkLte{} -> True; _ -> False) "<="++      <|>++      (\(tk, ws) -> Gt (pyTokenAnn tk) ws) <$>+      token ws (\case; TkGt{} -> True; _ -> False) ">"++      <|>++      (\(tk, ws) -> GtEq (pyTokenAnn tk) ws) <$>+      token ws (\case; TkGte{} -> True; _ -> False) ">="++      <|>++      (\(tk, ws) -> NotEq (pyTokenAnn tk) ws) <$>+      token ws (\case; TkBangEq{} -> True; _ -> False) "!="++    comparison = binOp compOp $ orExpr ws++yieldExpr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+yieldExpr ws =+  (\(tk, s) -> either (uncurry $ YieldFrom (pyTokenAnn tk) s) (Yield (pyTokenAnn tk) s)) <$>+  token ws (\case; TkYield{} -> True; _ -> False) "yield" <*>+  (fmap Left+     ((,) <$>+      (snd <$> token ws (\case; TkFrom{} -> True; _ -> False) "from") <*>+      expr ws)+     <|>+   Right <$> commaSep ws (expr ws))++lambda :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+lambda ws =+  (\(tk, s) -> Lambda (pyTokenAnn tk) s) <$>+  token ws (\case; TkLambda{} -> True; _ -> False) "lambda" <*>+  untypedParams ws <*>+  (MkColon . snd <$> token ws (\case; TkColon{} -> True; _ -> False) ":") <*>+  expr ws++lambdaNoCond :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+lambdaNoCond ws =+  (\(tk, s) -> Lambda (pyTokenAnn tk) s) <$>+  token ws (\case; TkLambda{} -> True; _ -> False) "lambda" <*>+  untypedParams ws <*>+  (MkColon . snd <$> token ws (\case; TkColon{} -> True; _ -> False) ":") <*>+  exprNoCond ws++exprNoCond :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+exprNoCond ws = orTest ws <|> lambdaNoCond ws++expr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+expr ws =+  (\a -> maybe a (\(b, c, d, e) -> Ternary (a ^. exprAnn) a b c d e)) <$>+  orTest ws <*>+  optional+    ((,,,) <$>+     (snd <$> token ws (\case; TkIf{} -> True; _ -> False) "if") <*>+     orTest ws <*>+     (snd <$> token ws (\case; TkElse{} -> True; _ -> False) "else") <*>+     expr ws)+  <|>+  lambda ws++rightParen+  :: MonadParsec e PyTokens m+  => m Whitespace+  -> m (PyToken SrcInfo, [Whitespace])+rightParen sp = token sp (\case; TkRightParen{} -> True; _ -> False) ")"++doubleStar+  :: MonadParsec e PyTokens m+  => m Whitespace+  -> m (PyToken SrcInfo, [Whitespace])+doubleStar sp = token sp (\case; TkDoubleStar{} -> True; _ -> False) "**"++orExpr :: MonadParsec e PyTokens m => m Whitespace -> m (Expr SrcInfo)+orExpr ws =+  binOp+    ((\(tk, ws) -> BitOr (pyTokenAnn tk) ws) <$>+     token ws (\case; TkPipe{} -> True; _ -> False) "|")+    xorExpr+  where+    xorExpr =+      binOp+        ((\(tk, ws) -> BitXor (pyTokenAnn tk) ws) <$>+         token ws (\case; TkCaret{} -> True; _ -> False) "^")+        andExpr++    andExpr =+      binOp+        ((\(tk, ws) -> BitAnd (pyTokenAnn tk) ws) <$>+         token ws (\case; TkAmpersand{} -> True; _ -> False) "&")+        shiftExpr++    shiftExpr =+      binOp+        ((\(tk, ws) -> ShiftLeft (pyTokenAnn tk) ws) <$>+         token ws (\case; TkShiftLeft{} -> True; _ -> False) "<<"++         <|>++         (\(tk, ws) -> ShiftRight (pyTokenAnn tk) ws) <$>+         token ws (\case; TkShiftRight{} -> True; _ -> False) ">>")+        arithExpr++    arithOp =+      (\(tk, ws) -> Plus (pyTokenAnn tk) ws) <$>+      token ws (\case; TkPlus{} -> True; _ -> False) "+"++      <|>++      (\(tk, ws) -> Minus (pyTokenAnn tk) ws) <$>+      token ws (\case; TkMinus{} -> True; _ -> False) "-"++    arithExpr = binOp arithOp term++    termOp =+      (\(tk, ws) -> Multiply (pyTokenAnn tk) ws) <$>+      star ws++      <|>++      (\(tk, ws) -> At (pyTokenAnn tk) ws) <$>+      token ws (\case; TkAt{} -> True; _ -> False) "@"++      <|>++      (\(tk, ws) -> Divide (pyTokenAnn tk) ws) <$>+      token ws (\case; TkSlash{} -> True; _ -> False) "/"++      <|>++      (\(tk, ws) -> FloorDivide (pyTokenAnn tk) ws) <$>+      token ws (\case; TkDoubleSlash{} -> True; _ -> False) "//"++      <|>++      (\(tk, ws) -> Percent (pyTokenAnn tk) ws) <$>+      token ws (\case; TkPercent{} -> True; _ -> False) "%"++    term = binOp termOp factor++    factor =+      ((\(tk, s) -> let ann = pyTokenAnn tk in UnOp ann (Negate ann s)) <$>+       token ws (\case; TkMinus{} -> True; _ -> False) "-"+       <|>+       (\(tk, s) -> let ann = pyTokenAnn tk in UnOp ann (Positive ann s)) <$>+       token ws (\case; TkPlus{} -> True; _ -> False) "+"+       <|>+       (\(tk, s) -> let ann = pyTokenAnn tk in UnOp ann (Complement ann s)) <$>+       token ws (\case; TkTilde{} -> True; _ -> False) "~") <*> factor+      <|>+      power++    powerOp =+      (\(tk, ws) -> Exp (pyTokenAnn tk) ws) <$>+      doubleStar ws++    power =+      (\a -> maybe a (uncurry $ BinOp (a ^. exprAnn) a)) <$>+      atomExpr <*>+      optional ((,) <$> powerOp <*> factor)++    subscript = do+      mex <- optional $ expr anySpace+      case mex of+        Nothing ->+          SubscriptSlice Nothing <$>+          (snd <$> colon anySpace) <*>+          optional (expr anySpace) <*>+          optional ((,) <$> (snd <$> colon anySpace) <*> optional (expr anySpace))+        Just ex -> do+          mws <- optional $ snd <$> colon anySpace+          case mws of+            Nothing -> pure $ SubscriptExpr ex+            Just ws ->+              SubscriptSlice (Just ex) ws <$>+              optional (expr anySpace) <*>+              optional ((,) <$> (snd <$> colon anySpace) <*> optional (expr anySpace))++    trailer =+      (\a b c -> Deref (c ^. exprAnn) c a b) <$>+      (snd <$> token ws (\case; TkDot{} -> True; _ -> False) ".") <*>+      identifier ws++      <|>++      (\a b c d -> Call (d ^. exprAnn) d a b c) <$>+      (snd <$> token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*>+      optional (commaSep1' anySpace arg) <*>+      (snd <$> rightParen ws)++      <|>++      (\a b c d -> Subscript (d ^. exprAnn) d a b c) <$>+      (snd <$> token anySpace (\case; TkLeftBracket{} -> True; _ -> False) "[") <*>+      commaSep1' anySpace subscript <*>+      (snd <$> token ws (\case; TkRightBracket{} -> True; _ -> False) "]")++    atomExpr =+      (\(mAwait, a) b ->+         let e = foldl' (&) a b+         in maybe e (\(tk, sp) -> Await (pyTokenAnn tk) sp e) mAwait) <$>+      try+        ((,) <$>+         optional (token ws (\case; TkIdent "await" _ -> True; _ -> False) "await") <*>+         atom) <*>+      many trailer+      <|>+      foldl' (&) <$> atom <*> many trailer++    parensOrUnit =+      (\(tk, s) maybeEx sps ->+       case maybeEx of+         Nothing -> Unit (pyTokenAnn tk) s sps+         Just ex -> Parens (pyTokenAnn tk) s ex sps) <$>+      token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(" <*>+      optional (yieldExpr anySpace <|> exprListComp anySpace) <*>+      (snd <$> rightParen ws)++    list =+      (\(tk, sp1) ->+         maybe (List (pyTokenAnn tk) sp1 Nothing) (\f -> f (pyTokenAnn tk) sp1)) <$>+      token anySpace (\case; TkLeftBracket{} -> True; _ -> False) "[" <*>+      optional+        ((\e a ann ws1 ->+          case a of+            Left (cf, cfs) -> ListComp ann ws1 (Comprehension (e ^. exprAnn) e cf cfs)+            Right Nothing -> List ann ws1 (Just $ CommaSepOne1' e Nothing)+            Right (Just (c, Nothing)) -> List ann ws1 (Just $ CommaSepOne1' e $ Just c)+            Right (Just (c, Just cs)) -> List ann ws1 (Just $ CommaSepMany1' e c cs)) <$>+        (expr anySpace <|> starExpr anySpace) <*>+        (Left <$>+        ((,) <$>+          compFor <*>+          many (Left <$> compFor <|> Right <$> compIf)) <|>+        Right <$>+        optional+          ((,) <$>+           (snd <$> comma anySpace) <*>+           optional (commaSep1' anySpace (expr anySpace <|> starExpr anySpace))))) <*>+      (snd <$> token ws (\case; TkRightBracket{} -> True; _ -> False) "]")++    doubleStarExpr ws =+      (\(tk, sp) -> DictUnpack (pyTokenAnn tk) sp) <$>+      doubleStar ws <*>+      orExpr ws++    dictItem =+      (\a -> DictItem (a ^. exprAnn) a) <$>+      expr anySpace <*>+      (snd <$> colon anySpace) <*>+      expr anySpace+      <|>+      doubleStarExpr anySpace++    compRHS = (,) <$> compFor <*> many (Left <$> compFor <|> Right <$> compIf)++    dictOrSet = do+      (a, ws1) <- token anySpace (\case; TkLeftBrace{} -> True; _ -> False) "{"+      let ann = pyTokenAnn a+      maybeExpr <-+        optional $+          Left . Left <$> expr anySpace <|>+          Left . Right <$> starExpr anySpace <|>+          Right <$> doubleStarExpr anySpace+      (case maybeExpr of+         Nothing -> pure $ Dict ann ws1 Nothing+         Just (Left (Left ex)) -> do+           maybeColon <-+             optional $ MkColon . snd <$> token anySpace (\case; TkColon{} -> True; _ -> False) ":"+           case maybeColon of+             Nothing ->+               -- The order of this choice matters because commaSepRest is implemented+               -- in a slightly odd way+               (\(c, d) -> SetComp ann ws1 (Comprehension (ex ^. exprAnn) ex c d)) <$>+               compRHS+               <|>++               (\(rest, final) -> Set ann ws1 ((ex, rest, final) ^. _CommaSep1')) <$>+               commaSepRest (expr anySpace <|> starExpr anySpace)+             Just clws ->+               (\ex2 a ->+                 let+                   dictItemAnn = ex ^. exprAnn+                   firstDictItem = DictItem dictItemAnn ex clws ex2+                 in+                 case a of+                   Left (c, d) ->+                     DictComp ann ws1 (Comprehension dictItemAnn firstDictItem c d)+                   Right (rest, final) ->+                     Dict ann ws1 (Just $ (firstDictItem, rest, final) ^. _CommaSep1')) <$>+               expr anySpace <*>+               (Left <$> compRHS <|> Right <$> commaSepRest dictItem)+         Just (Left (Right ex)) ->+           ((\(c, d) -> SetComp ann ws1 (Comprehension (ex ^. exprAnn) ex c d)) <$>+            compRHS++            <|>++            (\(rest, final) -> Set ann ws1 ((ex, rest, final) ^. _CommaSep1')) <$>+            commaSepRest (expr anySpace <|> starExpr anySpace))+         Just (Right ex) ->+           ((\(c, d) -> DictComp ann ws1 (Comprehension (_dictItemAnn ex) ex c d)) <$>+            compRHS++            <|>++            (\(rest, final) -> Dict ann ws1 (Just $ (ex, rest, final) ^. _CommaSep1')) <$>+            commaSepRest dictItem)) <*>++        (snd <$> token ws (\case; TkRightBrace{} -> True; _ -> False) "}")++    atom =+      dictOrSet <|>+      list <|>+      none ws <|>+      bool ws <|>+      ellipsis ws <|>+      integer ws <|>+      float ws <|>+      imag ws <|>+      stringOrBytes ws <|>+      Ident <$> identifier ws <|>+      parensOrUnit++simpleStatement :: MonadParsec e PyTokens m => m (SimpleStatement SrcInfo)+simpleStatement =+  returnSt <|>+  passSt <|>+  breakSt <|>+  continueSt <|>+  globalSt <|>+  nonlocalSt <|>+  delSt <|>+  importSt <|>+  raiseSt <|>+  exprOrAssignSt <|>+  yieldSt <|>+  assertSt+  where+    assertSt =+      (\(tk, s) -> Assert (pyTokenAnn tk) s) <$>+      token space (\case; TkAssert{} -> True; _ -> False) "assert" <*>+      expr space <*>+      optional ((,) <$> (snd <$> comma space) <*> expr space)++    yieldSt = (\a -> Expr (a ^. exprAnn) a) <$> yieldExpr space++    returnSt =+      (\(tkReturn, retSpaces) -> Return (pyTokenAnn tkReturn) retSpaces) <$>+      token space (\case; TkReturn{} -> True; _ -> False) "return" <*>+      optional (exprList space)++    passSt =+      uncurry (Pass . pyTokenAnn) <$>+      token space (\case; TkPass{} -> True; _ -> False) "pass"++    breakSt =+      uncurry (Break . pyTokenAnn) <$>+      token space (\case; TkBreak{} -> True; _ -> False) "break"++    continueSt =+      uncurry (Continue . pyTokenAnn) <$>+      token space (\case; TkContinue{} -> True; _ -> False) "continue"++    augAssign =+      (\(tk, s) -> MkAugAssign PlusEq (pyTokenAnn tk) s) <$>+      token space (\case; TkPlusEq{} -> True; _ -> False) "+="++      <|>++      (\(tk, s) -> MkAugAssign MinusEq (pyTokenAnn tk) s) <$>+      token space (\case; TkMinusEq{} -> True; _ -> False) "-="++      <|>++      (\(tk, s) -> MkAugAssign AtEq (pyTokenAnn tk) s) <$>+      token space (\case; TkAtEq{} -> True; _ -> False) "@="++      <|>++      (\(tk, s) -> MkAugAssign StarEq (pyTokenAnn tk) s) <$>+      token space (\case; TkStarEq{} -> True; _ -> False) "*="++      <|>++      (\(tk, s) -> MkAugAssign SlashEq (pyTokenAnn tk) s) <$>+      token space (\case; TkSlashEq{} -> True; _ -> False) "/="++      <|>++      (\(tk, s) -> MkAugAssign PercentEq (pyTokenAnn tk) s) <$>+      token space (\case; TkPercentEq{} -> True; _ -> False) "%="++      <|>++      (\(tk, s) -> MkAugAssign AmpersandEq (pyTokenAnn tk) s) <$>+      token space (\case; TkAmpersandEq{} -> True; _ -> False) "&="++      <|>++      (\(tk, s) -> MkAugAssign PipeEq (pyTokenAnn tk) s) <$>+      token space (\case; TkPipeEq{} -> True; _ -> False) "|="++      <|>++      (\(tk, s) -> MkAugAssign CaretEq (pyTokenAnn tk) s) <$>+      token space (\case; TkCaretEq{} -> True; _ -> False) "^="++      <|>++      (\(tk, s) -> MkAugAssign ShiftLeftEq (pyTokenAnn tk) s) <$>+      token space (\case; TkShiftLeftEq{} -> True; _ -> False) "<<="++      <|>++      (\(tk, s) -> MkAugAssign ShiftRightEq (pyTokenAnn tk) s) <$>+      token space (\case; TkShiftRightEq{} -> True; _ -> False) ">>="++      <|>++      (\(tk, s) -> MkAugAssign DoubleStarEq (pyTokenAnn tk) s) <$>+      token space (\case; TkDoubleStarEq{} -> True; _ -> False) "**="++      <|>++      (\(tk, s) -> MkAugAssign DoubleSlashEq (pyTokenAnn tk) s) <$>+      token space (\case; TkDoubleSlashEq{} -> True; _ -> False) "//="++    exprOrAssignSt =+      (\a ->+         maybe+           (Expr (a ^. exprAnn) a)+           (either+              (Assign (a ^. exprAnn) a)+              (uncurry $ AugAssign (a ^. exprAnn) a))) <$>+      exprOrStarList space <*>+      optional+        (Left <$>+         some1+           ((,) <$>+            (snd <$> equals space) <*>+            (yieldExpr space <|> exprOrStarList space))++           <|>++         Right <$> ((,) <$> augAssign <*> (yieldExpr space <|> exprList space)))++    globalSt =+      (\(tk, s) -> Global (pyTokenAnn tk) $ NonEmpty.fromList s) <$>+      token space (\case; TkGlobal{} -> True; _ -> False) "global" <*>+      commaSep1 space (identifier space)++    nonlocalSt =+      (\(tk, s) -> Nonlocal (pyTokenAnn tk) $ NonEmpty.fromList s) <$>+      token space (\case; TkNonlocal{} -> True; _ -> False) "nonlocal" <*>+      commaSep1 space (identifier space)++    delSt =+      (\(tk, s) -> Del (pyTokenAnn tk) s) <$>+      token space (\case; TkDel{} -> True; _ -> False) "del" <*>+      commaSep1' space (orExpr space)++    raiseSt =+      (\(tk, s) -> Raise (pyTokenAnn tk) s) <$>+      token space (\case; TkRaise{} -> True; _ -> False) "raise" <*>+      optional+        ((,) <$>+         expr space <*>+         optional+           ((,) <$>+            (snd <$> token space (\case; TkFrom{} -> True; _ -> False) "from") <*>+            expr space))++    importSt = importName <|> importFrom+      where+        moduleName =+          makeModuleName <$>+          identifier space <*>+          many+            ((,) <$>+             (snd <$> token space (\case; TkDot{} -> True; _ -> False) ".") <*>+             identifier space)++        importAs ws getAnn p =+          (\a -> ImportAs (getAnn a) a) <$>+          p <*>+          optional+            ((,) <$>+             (NonEmpty.fromList . snd <$> token ws (\case; TkAs{} -> True; _ -> False) "as") <*>+             identifier ws)++        importName =+          (\(tk, s) -> Import (pyTokenAnn tk) $ NonEmpty.fromList s) <$>+          token space (\case; TkImport{} -> True; _ -> False) "import" <*>+          commaSep1 space (importAs space _moduleNameAnn moduleName)++        dots =+          fmap concat . some $+          pure . snd <$> dot space++          <|>++          (\(_, ws) -> [MkDot [], MkDot [], MkDot ws]) <$>+          token space (\case; TkEllipsis{} -> True; _ -> False) "..."++        relativeModuleName =+          RelativeWithName [] <$> moduleName++          <|>++          (\a -> maybe (Relative $ NonEmpty.fromList a) (RelativeWithName a)) <$>+          dots <*>+          optional moduleName++        importTargets =+          (\(tk, s) -> ImportAll (pyTokenAnn tk) s) <$>+          star space++          <|>++          (\(tk, s) -> ImportSomeParens (pyTokenAnn tk) s) <$>+          token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(" <*>+          commaSep1' anySpace (importAs anySpace _identAnn (identifier anySpace)) <*>+          (snd <$> rightParen space)++          <|>++          (\a -> ImportSome (commaSep1Head a ^. importAsAnn) a) <$>+          commaSep1 space (importAs space _identAnn (identifier space))++        importFrom =+          (\(tk, s) -> From (pyTokenAnn tk) s) <$>+          token space (\case; TkFrom{} -> True; _ -> False) "from" <*>+          relativeModuleName <*>+          (snd <$> token space (\case; TkImport{} -> True; _ -> False) "import") <*>+          importTargets++sepBy1' :: MonadParsec e PyTokens m => m a -> m sep -> m (a, [(sep, a)], Maybe sep)+sepBy1' val sep = go+  where+    go =+      (\a b ->+         case b of+           Nothing -> (a, [], Nothing)+           Just (sc, b') ->+             case b' of+               Nothing -> (a, [], Just sc)+               Just (a', ls, sc') -> (a, (sc, a') : ls, sc')) <$>+      val <*>+      optional ((,) <$> sep <*> optional go)++smallStatement+  :: MonadParsec e PyTokens m+  => m (SmallStatement SrcInfo)+smallStatement =+  (\(a, b, c) d -> MkSmallStatement a b c d) <$>+  sepBy1' simpleStatement (snd <$> semicolon space) <*>+  optional comment <*>+  optional eol++statement+  :: (Alternative m, MonadParsec e PyTokens m)+  => m (Indents SrcInfo)+  -> Indents SrcInfo+  -> m (Statement SrcInfo)+statement pIndent indentBefore =+  -- It's important to parse compound statements first, because the 'async' keyword+  -- is actually an identifier and we'll have to backtrack+  CompoundStatement <$> compoundStatement pIndent indentBefore <|>+  SmallStatement indentBefore <$> smallStatement++blank :: MonadParsec e PyTokens m => m (Blank SrcInfo)+blank =+  withSrcInfo $+  (\b c a -> Blank a b c) <$>+  some space <*>+  optional comment++  <|>++  (\b a -> Blank a [] b) <$> optional comment++suite :: MonadParsec e PyTokens m => m (Suite SrcInfo)+suite =+  (\(tk, s) ->+     either+       (SuiteOne (pyTokenAnn tk) s)+       (\(a, b,c ) -> SuiteMany (pyTokenAnn tk) s a b c)) <$>+  colon space <*>+  (Left <$> smallStatement++    <|>++   (fmap Right $+    (,,) <$>+    optional comment <*>+    eol <*>+    (Block <$>+     many ((,) <$> blank <*> eol) <*>+     (statement level =<< indent) <*>+     many (line level)) <*+    dedent))+  where++    line i =+      Left <$> ((,) <$> blank <*> eol) <|>+      Right <$> (statement level =<< i)++commaSep :: MonadParsec e PyTokens m => m Whitespace -> m a -> m (CommaSep a)+commaSep ws pa =+  (\a -> maybe (CommaSepOne a) (uncurry $ CommaSepMany a)) <$>+  pa <*>+  optional ((,) <$> (snd <$> comma ws) <*> commaSep ws pa)++  <|>++  pure CommaSepNone++commaSep1 :: MonadParsec e PyTokens m => m Whitespace -> m a -> m (CommaSep1 a)+commaSep1 ws val = go+  where+    go =+      (\a -> maybe (CommaSepOne1 a) (uncurry $ CommaSepMany1 a)) <$>+      val <*>+      optional ((,) <$> (snd <$> comma ws) <*> go)++commaSep1' :: MonadParsec e PyTokens m => m Whitespace -> m a -> m (CommaSep1' a)+commaSep1' ws pa =+  (\(a, b, c) -> from a b c) <$> sepBy1' pa (snd <$> comma ws)+  where+    from a [] b = CommaSepOne1' a b+    from a ((b, c) : bs) d = CommaSepMany1' a b $ from c bs d++someParams+  :: MonadParsec e PyTokens m+  => m (Param SrcInfo)+  -> m (Param SrcInfo)+  -> m (Param SrcInfo)+  -> m (CommaSep (Param SrcInfo))+someParams paramPositional paramStar paramDoubleStar =+  fmap (view _CommaSep) . optional $++  (\a b c ->+     case c of+       Just (d, e) ->+         case e of+           Nothing -> (a, b, Just d)+           Just f ->+             case f of+               Left (g, h, i) -> (a, b ++ (d, g) : maybe h (snoc h) i, Nothing)+               Right g -> (a, snoc b (d, g), Nothing)+       Nothing -> (a, b, Nothing)) <$>++  paramPositional <*>++  many commaPositional <*>++  optional+    ((,) <$>+     (snd <$> comma anySpace) <*>+     optional+       (Left <$>+        ((,,) <$> paramStar <*> many commaPositional <*> optional commaDoubleStar)++        <|>++        Right <$> paramDoubleStar))++  <|>++  (\a b -> (a, b, Nothing)) <$>+  paramStar <*>+  ((\a -> maybe a (a `snoc`)) <$>+   many commaPositional <*>+   optional commaDoubleStar)++  <|>++  (\a -> (a, [], Nothing)) <$> paramDoubleStar++  where+    commaPositional =+      try+        ((,) <$>+         fmap snd (comma anySpace) <*+         notFollowedBy+           (star anySpace <|>+            doubleStar anySpace <|>+            rightParen space)) <*>+      paramPositional++    commaDoubleStar =+      (,) <$> (snd <$> comma anySpace) <*> paramDoubleStar++upPositional :: MonadParsec e PyTokens m => m Whitespace -> m (Param SrcInfo)+upPositional ws =+  (\a ->+    maybe+      (PositionalParam (_identAnn a) a Nothing)+      (uncurry $ KeywordParam (_identAnn a) a Nothing)) <$>+  identifier ws <*>+  optional+    ((,) <$>+    (snd <$> token ws (\case; TkEq{} -> True; _ -> False) "=") <*>+    expr ws)++upStar :: MonadParsec e PyTokens m => m Whitespace -> m (Param SrcInfo)+upStar ws =+  (\(a, b) ->+    maybe+      (UnnamedStarParam (pyTokenAnn a) b)+      (uncurry $ StarParam (pyTokenAnn a) b)) <$>+  star ws <*>+  optional ((\a -> (a, Nothing)) <$> identifier ws)++upDoubleStar :: MonadParsec e PyTokens m => m Whitespace -> m (Param SrcInfo)+upDoubleStar ws =+  (\(a, b) c -> DoubleStarParam (pyTokenAnn a) b c Nothing) <$>+  doubleStar ws <*>+  identifier ws++untypedParams+  :: MonadParsec e PyTokens m+  => m Whitespace+  -> m (CommaSep (Param SrcInfo))+untypedParams ws = someParams (upPositional ws) (upStar ws) (upDoubleStar ws)++tyAnn :: MonadParsec e PyTokens m => m (Colon, Expr SrcInfo)+tyAnn =+  (,) <$>+  (MkColon . snd <$> token anySpace (\case; TkColon{} -> True; _ -> False) ":") <*>+  expr anySpace++tpPositional :: MonadParsec e PyTokens m => m (Param SrcInfo)+tpPositional =+  (\a b ->+    maybe+      (PositionalParam (_identAnn a) a b)+      (uncurry $ KeywordParam (_identAnn a) a b)) <$>+  identifier anySpace <*>+  optional tyAnn <*>+  optional+    ((,) <$>+    (snd <$> token anySpace (\case; TkEq{} -> True; _ -> False) "=") <*>+    expr anySpace)++tpStar :: MonadParsec e PyTokens m => m (Param SrcInfo)+tpStar =+  (\(a, b) ->+    maybe+      (UnnamedStarParam (pyTokenAnn a) b)+      (uncurry $ StarParam (pyTokenAnn a) b)) <$>+  star anySpace <*>+  optional ((,) <$> identifier anySpace <*> optional tyAnn)++tpDoubleStar :: MonadParsec e PyTokens m => m (Param SrcInfo)+tpDoubleStar =+  (\(a, b) -> DoubleStarParam (pyTokenAnn a) b) <$>+  doubleStar anySpace <*>+  identifier anySpace <*>+  optional tyAnn++typedParams :: MonadParsec e PyTokens m => m (CommaSep (Param SrcInfo))+typedParams = someParams tpPositional tpStar tpDoubleStar++arg :: MonadParsec e PyTokens m => m (Arg SrcInfo)+arg =+  (do+      e <- exprComp anySpace+      case e of+        Ident ident -> do+          eqSpaces <-+            optional $ snd <$> token anySpace (\case; TkEq{} -> True; _ -> False) "="+          case eqSpaces of+            Nothing -> pure $ PositionalArg (e ^. exprAnn) e+            Just s -> KeywordArg (e ^. exprAnn) ident s <$> expr anySpace+        _ -> pure $ PositionalArg (e ^. exprAnn) e)++  <|>++  (\a -> PositionalArg (a ^. exprAnn) a) <$> expr anySpace++  <|>++  (\(a, b) -> StarArg (pyTokenAnn a) b) <$>+  star anySpace <*>+  expr anySpace++  <|>++  (\(a, b) -> DoubleStarArg (pyTokenAnn a) b) <$>+  doubleStar anySpace <*>+  expr anySpace++decoratorValue :: MonadParsec e PyTokens m => m (Expr SrcInfo)+decoratorValue = do+  id1 <- identifier space+  ids <-+    many+      ((,) <$>+       (snd <$> token space (\case; TkDot{} -> True; _ -> False) ".") <*>+       identifier space)+  args <-+    optional $+    (,,) <$>+    (snd <$> token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*>+    optional (commaSep1' anySpace arg) <*>+    (snd <$> rightParen space)+  let+    derefs =+      foldl+        (\b (ws, a) -> Deref (b ^. exprAnn) b ws a)+        (Ident id1)+        ids+  pure $+    case args of+      Nothing -> derefs+      Just (l, x, r) -> Call (derefs ^. exprAnn) derefs l x r++decorator+  :: MonadParsec e PyTokens m+  => Indents SrcInfo+  -> m (Decorator SrcInfo)+decorator indentBefore =+  (\(tk, spcs) a b -> Decorator (pyTokenAnn tk) indentBefore spcs a b) <$>+  at space <*>+  decoratorValue <*>+  optional comment <*>+  eol <*>+  many ((,) <$> blank <*> eol)++decorators+  :: MonadParsec e PyTokens m+  => m (Indents SrcInfo)+  -> Indents SrcInfo+  -> m [Decorator SrcInfo]+decorators pIndent indentBefore =+  (:) <$>+  decorator indentBefore <*>+  many (try i >>= decorator)+  where+    i =+      pIndent <*+      lookAhead (token space (\case; TkAt{} -> True; _ -> False) "@")++compoundStatement+  :: MonadParsec e PyTokens m+  => m (Indents SrcInfo)+  -> Indents SrcInfo+  -> m (CompoundStatement SrcInfo)+compoundStatement pIndent indentBefore =+  ifSt <|>+  whileSt <|>+  trySt <|>+  decorated <|>+  asyncSt <|>+  classSt indentBefore [] <|>+  fundef indentBefore Nothing [] <|>+  withSt Nothing <|>+  forSt Nothing+  where+    decorated = do+      ds <- decorators pIndent indentBefore+      i <- pIndent+      (do; a <- doAsync; fundef i (Just a) ds) <|>+        fundef i Nothing ds <|>+        classSt i ds++    classSt ib decs =+      (\(tk, s) a b c ->+        ClassDef+          (pyTokenAnn tk)+          decs+          ib+          (NonEmpty.fromList s) a b c) <$>+      token space (\case; TkClass{} -> True; _ -> False) "class" <*>+      identifier space <*>+      optional+        ((,,) <$>+         (snd <$> token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*>+         optional (commaSep1' anySpace arg) <*>+         (snd <$> rightParen space)) <*>+      suite++    ifSt =+      (\(tk, s) a b c d -> If (pyTokenAnn tk) indentBefore s a b c d) <$>+      token space (\case; TkIf{} -> True; _ -> False) "if" <*>+      expr space <*>+      suite <*>+      many+        (try+           ((,,,) <$>+            pIndent <*>+            (snd <$> token space (\case; TkElif{} -> True; _ -> False) "elif")) <*>+         expr space <*>+         suite) <*>+      optional+        (try+           ((,,) <$>+            pIndent <*>+            (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*>+         suite)++    whileSt =+      (\(tk, s) a b -> While (pyTokenAnn tk) indentBefore s a b) <$>+      token space (\case; TkWhile{} -> True; _ -> False) "while" <*>+      expr space <*>+      suite <*>+      optional+        (try+           ((,,) <$>+            pIndent <*>+            (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*>+         suite)++    exceptAs =+      (\a -> ExceptAs (a ^. exprAnn) a) <$>+      expr space <*>+      optional+        ((,) <$>+         (snd <$> token space (\case; TkAs{} -> True; _ -> False) "as") <*>+         identifier space)++    trySt =+      (\(tk, s) a d ->+         case d of+           Left (e, f, g) -> TryFinally (pyTokenAnn tk) indentBefore s a e f g+           Right (e, f, g) -> TryExcept (pyTokenAnn tk) indentBefore s a e f g) <$>+      token space (\case; TkTry{} -> True; _ -> False) "try" <*>+      suite <*>+      (fmap Left+         (try+            ((,,) <$>+             pIndent <*>+             (snd <$> token space (\case; TkFinally{} -> True; _ -> False) "finally")) <*>+          suite)++        <|>++        fmap Right+          ((,,) <$>+           some1+             (try+                ((,,,) <$>+                 pIndent <*>+                 (snd <$> token space (\case; TkExcept{} -> True; _ -> False) "except")) <*>+              optional exceptAs <*>+              suite) <*>+           optional+             (try+                ((,,) <$>+                 pIndent <*>+                 (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*>+              suite) <*>+           optional+             (try+                ((,,) <$>+                 pIndent <*>+                 (snd <$> token space (\case; TkFinally{} -> True; _ -> False) "finally")) <*>+              suite)))++    doAsync = token space (\case; TkIdent "async" _ -> True; _ -> False) "async"++    asyncSt = do+      a <-+        try $+        doAsync <*+        lookAhead+          (token space (\case; TkDef{} -> True; _ -> False) "def" <|>+           token space (\case; TkWith{} -> True; _ -> False) "with" <|>+           token space (\case; TkFor{} -> True; _ -> False) "for")+      fundef indentBefore (Just a) [] <|>+        withSt (Just a) <|>+        forSt (Just a)++    fundef ib async decs =+      (\(tkDef, defSpaces) a b c d e f ->+         Fundef+         (maybe (pyTokenAnn tkDef) (pyTokenAnn . fst) async)+         decs+         ib+         (NonEmpty.fromList . snd <$> async)+         (NonEmpty.fromList defSpaces)+         a b c d e f) <$>+      token space (\case; TkDef{} -> True; _ -> False) "def" <*>+      identifier space <*>+      fmap snd (token anySpace (\case; TkLeftParen{} -> True; _ -> False) "(") <*>+      typedParams <*>+      fmap snd (rightParen space) <*>+      optional+        ((,) <$>+         (snd <$> token space (\case; TkRightArrow{} -> True; _ -> False) "->") <*>+         expr space) <*>+      suite++    withSt async =+      (\(tk, s) a b ->+          With+            (maybe (pyTokenAnn tk) (pyTokenAnn . fst) async)+            indentBefore+            (NonEmpty.fromList . snd <$> async)+            s a b) <$>+      token space (\case; TkWith{} -> True; _ -> False) "with" <*>+      commaSep1+        space+        ((\a -> WithItem (a ^. exprAnn) a) <$>+         expr space <*>+         optional+           ((,) <$>+            (snd <$> token space (\case; TkAs{} -> True; _ -> False) "as") <*>+            orExpr space)) <*>+      suite++    forSt async =+      (\(tk, s) a b c d e ->+        For+          (maybe (pyTokenAnn tk) (pyTokenAnn . fst) async)+          indentBefore+          (NonEmpty.fromList . snd <$> async)+          s a b c d e) <$>+      token space (\case; TkFor{} -> True; _ -> False) "for" <*>+      orExprList space <*>+      (snd <$> token space (\case; TkIn{} -> True; _ -> False) "in") <*>+      commaSep1' space (expr space) <*>+      suite <*>+      optional+        (try+           ((,,) <$>+            pIndent <*>+            (snd <$> token space (\case; TkElse{} -> True; _ -> False) "else")) <*>+         suite)++module_ :: MonadParsec e PyTokens m => m (Module SrcInfo)+module_ =+  ModuleStatement <$> (statement tlIndent =<< tlIndent) <*> module_++  <|>++  (\bl rest ->+     case rest of+       Left (nl, md) -> ModuleBlank bl nl md+       Right{} -> ModuleBlankFinal bl) <$>+  blank <*>+  (Left <$> ((,) <$> newline <*> module_) <|> Right <$> eof)++  <|>++  ModuleEmpty <$ eof++  where+    tlIndent = level <|> withSrcInfo (pure $ Indents [])
+ src/Language/Python/Internal/Render.hs view
@@ -0,0 +1,1639 @@+{-# language GeneralizedNewtypeDeriving #-}+{-# language FlexibleInstances, MultiParamTypeClasses #-}+{-# language OverloadedStrings #-}++{-|+Module      : Language.Python.Internal.Render+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Internal.Render+  ( -- * Common Functions+    showModule, showStatement, showExpr+    -- * Rendering+  , RenderOutput, showRenderOutput, singleton+  , renderModule, renderStatement, renderExpr+    -- * Miscellany+  , showQuoteType, showStringPrefix, showBytesPrefix, showToken, showTokens+  , expandIndents, whitespaceTokens, commentTokens+  , parens, braces, brackets+  , renderWhitespace, renderCommaSep, renderCommaSep1, renderCommaSep1'+  , renderIdent, renderComment, renderModuleName, renderDot, renderRelativeModuleName+  , renderImportAs, renderImportTargets, renderSimpleStatement, renderCompoundStatement+  , renderBlock, renderIndent, renderIndents, renderExceptAs, renderArg, renderParam+  , renderParams, renderCompFor, renderCompIf, renderComprehension, renderBinOp, renderUnOp+  , renderSubscript, renderPyChars, escapeChars, intToHex+  )+where++import Control.Lens.Cons (_init, _last)+import Control.Lens.Fold ((^..), folded, traverseOf_)+import Control.Lens.Getter ((^.))+import Control.Lens.Review ((#))+import Control.Lens.Setter ((.~))+import Control.Monad.Writer.Strict (Writer, execWriter, writer)+import Control.Monad.Reader (ReaderT, runReaderT, local, ask)+import Data.Bifoldable (bitraverse_, bitraverse_)+import Data.Char (ord)+import Data.Digit.Char (charHeXaDeCiMaL, charOctal)+import Data.Digit.Hexadecimal.MixedCase (HeXDigit(..))+import Data.DList (DList)+import Data.Function ((&))+import Data.Foldable (toList, traverse_)+import Data.Maybe (isNothing)+import Data.Semigroup (Semigroup(..))+import Data.Text (Text)++import qualified Data.DList as DList+import qualified Data.List.NonEmpty as NonEmpty+import qualified Data.Text as Text+import qualified Data.Text.Lazy as Lazy+import qualified Data.Text.Lazy.Builder as Builder++import Language.Python.Internal.Render.Correction+import Language.Python.Internal.Token (PyToken(..))+import Language.Python.Syntax.AugAssign+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Comment+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Import+import Language.Python.Syntax.Module+import Language.Python.Syntax.ModuleNames+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Operator.Binary+import Language.Python.Syntax.Operator.Unary+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace++-- | A 'RenderOutput' is an intermediate form used during rendering+-- with efficient concatenation+newtype RenderOutput a+  = RenderOutput+  { unRenderOutput+    :: ReaderT+         -- Is the thing we're rendering followed by an optional+         -- newline? (as opposed to mandatory newline)+         --+         -- This is because the AST may be missing critical newlines+         -- and we supply them during rendering+         Bool+         (Writer (DList (PyToken ())))+         a+  } deriving (Functor, Applicative, Monad)++-- | Treats the input as a terminating statement (does not cause additional newlines to+-- be inserted)+final :: RenderOutput a -> RenderOutput a+final = id++-- | Treats the input as a non-terminating statement (causes additional newlines to be+-- inserted)+notFinal :: RenderOutput a -> RenderOutput a+notFinal (RenderOutput a) = RenderOutput $ local (const False) a++-- | Are we inside a terminating or non-terminating context?+isFinal :: RenderOutput Bool+isFinal = RenderOutput ask++-- | Render a single token as a 'RenderOutput'+singleton :: PyToken () -> RenderOutput ()+singleton a = RenderOutput $ writer ((), DList.singleton a)++-- | Run a 'RenderOutput' to produce a final 'Text'.+--+-- These 'Text's should then not be appended any more. All appending should+-- be done during the 'RenderOutput' phase.+showRenderOutput :: RenderOutput a -> Text+showRenderOutput =+  Lazy.toStrict .+  Builder.toLazyText .+  foldMap (Builder.fromText . showToken) .+  correctSpaces showToken .+  correctNewlines .+  DList.toList .+  execWriter .+  flip runReaderT True .+  unRenderOutput++renderComment :: Comment a -> RenderOutput ()+renderComment = traverse_ singleton . commentTokens++commentTokens :: Comment a -> [PyToken ()]+commentTokens c = [TkComment $ () <$ c]++showComment :: Comment a -> Text+showComment (MkComment _ s) = Text.pack $ "#" <> s++between :: RenderOutput l -> RenderOutput r -> RenderOutput a -> RenderOutput a+between l r m = l *> m <* r++parens :: RenderOutput a -> RenderOutput a+parens = between (singleton $ TkLeftParen ()) (singleton $ TkRightParen ())++brackets :: RenderOutput a -> RenderOutput a+brackets = between (singleton $ TkLeftBracket ()) (singleton $ TkRightBracket ())++braces :: RenderOutput a -> RenderOutput a+braces = between (singleton $ TkLeftBrace ()) (singleton $ TkRightBrace ())++-- | Parenthesise a term, but put its trailing whitespace *outside* the parens+parensDistTWS+  :: HasTrailingWhitespace s+  => (s -> RenderOutput ())+  -> s -> RenderOutput ()+parensDistTWS f a = do+  parens $ f (a & trailingWhitespace .~ [])+  traverse_ renderWhitespace (a ^. trailingWhitespace)++parensTuple :: Expr v a -> RenderOutput ()+parensTuple e =+  case e of+    Tuple{} -> parensDistTWS renderExpr e+    _ -> renderExpr e++parensGenerator :: Expr v a -> RenderOutput ()+parensGenerator e =+  case e of+    Generator{} -> parensDistTWS renderExpr e+    _ -> renderExpr e++parensTupleGenerator :: Expr v a -> RenderOutput ()+parensTupleGenerator e =+  case e of+    Tuple{} -> parensDistTWS renderExpr e+    Generator{} -> parensDistTWS renderExpr e+    _ -> renderExpr e++escapeChars :: [(Char, Char)]+escapeChars =+  [ ('\\', '\\')+  , ('"', '"')+  , ('\a', 'a')+  , ('\b', 'b')+  , ('\f', 'f')+  , ('\n', 'n')+  , ('\r', 'r')+  , ('\t', 't')+  , ('\v', 'v')+  ]++intToHex :: Int -> Text+intToHex n = Text.pack $ go n []+  where+    go 0 = (++"0")+    go 1 = (++"1")+    go 2 = (++"2")+    go 3 = (++"3")+    go 4 = (++"4")+    go 5 = (++"5")+    go 6 = (++"6")+    go 7 = (++"7")+    go 8 = (++"8")+    go 9 = (++"9")+    go 10 = (++"A")+    go 11 = (++"B")+    go 12 = (++"C")+    go 13 = (++"D")+    go 14 = (++"E")+    go 15 = (++"F")+    go b = let (q, r) = quotRem b 16 in go r . go q++intToHexH :: Int -> [HeXDigit]+intToHexH n = go n []+  where+    go 0 = (++[HeXDigit0])+    go 1 = (++[HeXDigit1])+    go 2 = (++[HeXDigit2])+    go 3 = (++[HeXDigit3])+    go 4 = (++[HeXDigit4])+    go 5 = (++[HeXDigit5])+    go 6 = (++[HeXDigit6])+    go 7 = (++[HeXDigit7])+    go 8 = (++[HeXDigit8])+    go 9 = (++[HeXDigit9])+    go 10 = (++[HeXDigitA])+    go 11 = (++[HeXDigitB])+    go 12 = (++[HeXDigitC])+    go 13 = (++[HeXDigitD])+    go 14 = (++[HeXDigitE])+    go 15 = (++[HeXDigitF])+    go b = let (q, r) = quotRem b 16 in go r . go q++renderPyCharsWithCorrection+  :: (QuoteType -> StringType -> [PyChar] -> [PyChar])+  -> QuoteType+  -> StringType+  -> [PyChar] -> Text+renderPyCharsWithCorrection c qt st = Text.pack . go . c qt st+  where+    go s =+      case s of+        [] -> ""+        Char_newline : cs -> "\\newline" <> go cs+        Char_octal1 a : cs ->+          "\\" <>+          [charOctal # a] <>+          go cs+        Char_octal2 a b : cs ->+          "\\" <>+          [charOctal # a, charOctal # b] <>+          go cs+        Char_octal3 a b c : cs ->+          "\\" <>+          [charOctal # a, charOctal # b, charOctal # c] <>+          go cs+        Char_hex a b : cs ->+          "\\x" <> [charHeXaDeCiMaL # a, charHeXaDeCiMaL # b] <> go cs+        Char_uni16 a b c d : cs ->+          "\\u" <>+          [ charHeXaDeCiMaL # a+          , charHeXaDeCiMaL # b+          , charHeXaDeCiMaL # c+          , charHeXaDeCiMaL # d+          ] <>+          go cs+        Char_uni32 a b c d e f g h : cs ->+          "\\u" <>+          [ charHeXaDeCiMaL # a+          , charHeXaDeCiMaL # b+          , charHeXaDeCiMaL # c+          , charHeXaDeCiMaL # d+          , charHeXaDeCiMaL # e+          , charHeXaDeCiMaL # f+          , charHeXaDeCiMaL # g+          , charHeXaDeCiMaL # h+          ] <>+          go cs+        Char_esc_bslash : cs -> '\\' : '\\' : go cs+        Char_esc_singlequote : cs -> '\\' : '\'' : go cs+        Char_esc_doublequote : cs -> '\\' : '"' : go cs+        Char_esc_a : cs -> '\\' : 'a' : go cs+        Char_esc_b : cs -> '\\' : 'b' : go cs+        Char_esc_f : cs -> '\\' : 'f' : go cs+        Char_esc_n : cs -> '\\' : 'n' : go cs+        Char_esc_r : cs -> '\\' : 'r' : go cs+        Char_esc_t : cs -> '\\' : 't' : go cs+        Char_esc_v : cs -> '\\' : 'v' : go cs+        Char_lit c : cs ->+          case st of+            LongString -> c : go cs+            ShortString ->+              case c of+                '\r' -> go $ Char_esc_r : cs+                '\n' -> go $ Char_esc_n : cs+                _ -> c : go cs++renderPyChars :: QuoteType -> StringType -> [PyChar] -> Text+renderPyChars =+  renderPyCharsWithCorrection $+  \qt st ->+    case st of+      LongString ->+        correctBackslashes . correctBackslashEscapes .+        correctInitialFinalQuotesLong qt+      ShortString ->+        correctBackslashes . correctBackslashEscapes .+        correctQuotes qt++renderRawPyChars :: QuoteType -> StringType -> [PyChar] -> Text+renderRawPyChars =+  renderPyCharsWithCorrection $+  \qt st ->+    case st of+      LongString ->+        correctInitialFinalQuotesLongRaw qt .+        correctBackslashEscapesRaw .+        correctBackslashesRaw+      ShortString ->+        correctBackslashEscapesRaw . correctBackslashesRaw .+        correctQuotesRaw qt++renderPyCharsBytesWithCorrection+  :: (QuoteType -> StringType -> [PyChar] -> [PyChar])+  -> QuoteType+  -> StringType+  -> [PyChar] -> Text+renderPyCharsBytesWithCorrection c qt st = Text.pack . go . c qt st+  where+    go s =+      case s of+        [] -> ""+        Char_newline : cs -> "\\newline" <> go cs+        Char_octal1 a  : cs ->+          "\\" <>+          [charOctal # a] <>+          go cs+        Char_octal2 a b : cs ->+          "\\" <>+          [charOctal # a, charOctal # b] <>+          go cs+        Char_octal3 a b c : cs ->+          "\\" <>+          [charOctal # a, charOctal # b, charOctal # c] <>+          go cs+        Char_hex a b : cs ->+          "\\x" <> [charHeXaDeCiMaL # a, charHeXaDeCiMaL # b] <> go cs+        Char_uni16 a b c d : cs ->+          "\\u" <>+          [ charHeXaDeCiMaL # a+          , charHeXaDeCiMaL # b+          , charHeXaDeCiMaL # c+          , charHeXaDeCiMaL # d+          ] <>+          go cs+        Char_uni32 a b c d e f g h : cs ->+          "\\u" <>+          [ charHeXaDeCiMaL # a+          , charHeXaDeCiMaL # b+          , charHeXaDeCiMaL # c+          , charHeXaDeCiMaL # d+          , charHeXaDeCiMaL # e+          , charHeXaDeCiMaL # f+          , charHeXaDeCiMaL # g+          , charHeXaDeCiMaL # h+          ] <>+          go cs+        Char_esc_bslash : cs -> '\\' : '\\' : go cs+        Char_esc_singlequote : cs -> '\\' : '\'' : go cs+        Char_esc_doublequote : cs -> '\\' : '"' : go cs+        Char_esc_a : cs -> '\\' : 'a' : go cs+        Char_esc_b : cs -> '\\' : 'b' : go cs+        Char_esc_f : cs -> '\\' : 'f' : go cs+        Char_esc_n : cs -> '\\' : 'n' : go cs+        Char_esc_r : cs -> '\\' : 'r' : go cs+        Char_esc_t : cs -> '\\' : 't' : go cs+        Char_esc_v : cs -> '\\' : 'v' : go cs+        Char_lit c : cs+          | o <- ord c, o > 127 ->+            let+              h = intToHexH o+            in+            case replicate (8 - length h) HeXDigit0 <> h of+              [a, b, c, d, e, f, g, h] -> go $ Char_uni32 a b c d e f g h : cs+              _ -> error $ "character " <> show c <> " out of unicode range"+          | otherwise ->+              case st of+                LongString -> c : go cs+                ShortString ->+                  case c of+                    '\r' -> go $ Char_esc_r : cs+                    '\n' -> go $ Char_esc_n : cs+                    _ -> c : go cs++renderPyCharsBytes :: QuoteType -> StringType -> [PyChar] -> Text+renderPyCharsBytes =+  renderPyCharsBytesWithCorrection $+  \qt st ->+  case st of+    LongString ->+      correctBackslashes . correctBackslashEscapes . correctInitialFinalQuotesLong qt+    ShortString ->+      correctBackslashes . correctBackslashEscapes . correctQuotes qt++renderRawPyCharsBytes :: QuoteType -> StringType -> [PyChar] -> Text+renderRawPyCharsBytes =+  renderPyCharsBytesWithCorrection $+  \qt st ->+    case st of+      LongString ->+        correctInitialFinalQuotesLongRaw qt .+        correctBackslashEscapesRaw .+        correctBackslashesRaw+      ShortString ->+        correctBackslashEscapesRaw . correctBackslashesRaw .+        correctQuotesRaw qt++showTokens :: [PyToken a] -> Text+showTokens =+  Lazy.toStrict .+  Builder.toLazyText .+  foldMap (Builder.fromText . showToken . (() <$)) .+  (expandIndents =<<)++expandIndents :: PyToken a -> [PyToken ()]+expandIndents (TkIndent _ i) =+  (i ^.. indentsValue.folded.indentWhitespaces.folded) >>=+  whitespaceTokens +expandIndents (TkLevel _ i) =+  (i ^.. indentsValue.folded.indentWhitespaces.folded) >>=+  whitespaceTokens+expandIndents TkDedent{} = []+expandIndents a = pure $ () <$ a++showToken :: PyToken a -> Text+showToken t =+  case t of+    TkIndent{} -> error "trying to show indent token"+    TkLevel{} -> error "trying to show level token"+    TkDedent{} -> error "trying to show dedent token"+    TkIf{} -> "if"+    TkElse{} -> "else"+    TkElif{} -> "elif"+    TkWhile{} -> "while"+    TkAssert{} -> "assert"+    TkDef{} -> "def"+    TkReturn{} -> "return"+    TkPass{} -> "pass"+    TkBreak{} -> "break"+    TkContinue{} -> "continue"+    TkTrue{} -> "True"+    TkFalse{} -> "False"+    TkNone{} -> "None"+    TkEllipsis{} -> "..."+    TkOr{} -> "or"+    TkAnd{} -> "and"+    TkIs{} -> "is"+    TkNot{} -> "not"+    TkGlobal{} -> "global"+    TkNonlocal{} -> "nonlocal"+    TkDel{} -> "del"+    TkLambda{} -> "lambda"+    TkImport{} -> "import"+    TkFrom{} -> "from"+    TkAs{} -> "as"+    TkRaise{} -> "raise"+    TkTry{} -> "try"+    TkExcept{} -> "except"+    TkFinally{} -> "finally"+    TkClass{} -> "class"+    TkRightArrow{} -> "->"+    TkWith{} -> "with"+    TkFor{} -> "for"+    TkIn{} -> "in"+    TkYield{} -> "yield"+    TkInt i -> showIntLiteral i+    TkFloat i -> showFloatLiteral i+    TkImag i -> showImagLiteral i+    TkIdent s _ -> Text.pack s+    TkString sp st qt s _ ->+      let+        quote =+          Text.pack $+          (case st of; LongString -> replicate 3; ShortString -> pure) (showQuoteType qt)+      in+        foldMap showStringPrefix sp <>+        quote <>+        renderPyChars qt st s <>+        quote+    TkBytes sp st qt s _ ->+      let+        quote =+          Text.pack $+          (case st of; LongString -> replicate 3; ShortString -> pure) (showQuoteType qt)+      in+        showBytesPrefix sp <>+        quote <>+        renderPyCharsBytes qt st s <>+        quote+    TkRawString sp st qt s _ ->+      let+        quote =+          case st of+            LongString -> Text.pack . replicate 3 $ showQuoteType qt+            ShortString -> Text.singleton $ showQuoteType qt+      in+        showRawStringPrefix sp <>+        quote <>+        renderRawPyChars qt st s <>+        quote+    TkRawBytes sp st qt s _ ->+      let+        quote =+          case st of+            LongString -> Text.pack . replicate 3 $ showQuoteType qt+            ShortString -> Text.singleton $ showQuoteType qt+      in+        showRawBytesPrefix sp <>+        quote <>+        renderRawPyCharsBytes qt st s <>+        quote+    TkSpace{} -> " "+    TkTab{} -> "\t"+    TkNewline nl _ ->+      case nl of+        CR -> "\r"+        LF -> "\n"+        CRLF -> "\r\n"+    TkLeftBracket{} -> "["+    TkRightBracket{} -> "]"+    TkLeftParen{} -> "("+    TkRightParen{} -> ")"+    TkLeftBrace{} -> "{"+    TkRightBrace{} -> "}"+    TkLt{} -> "<"+    TkLte{} -> "<="+    TkEq{} -> "="+    TkDoubleEq{}-> "=="+    TkBangEq{}-> "!="+    TkGt{} -> ">"+    TkGte{} -> ">="+    TkContinued nl _ ->+      "\\" <>+      case nl of+        CR -> "\r"+        LF -> "\n"+        CRLF -> "\r\n"+    TkColon{} -> ":"+    TkSemicolon{} -> ";"+    TkComma{} -> ","+    TkDot{} -> "."+    TkPlus{} -> "+"+    TkMinus{} -> "-"+    TkTilde{} -> "~"+    TkComment c -> showComment c+    TkStar{} -> "*"+    TkDoubleStar{} -> "**"+    TkSlash{} -> "/"+    TkDoubleSlash{} -> "//"+    TkPercent{} -> "%"+    TkShiftLeft{} -> "<<"+    TkShiftRight{} -> ">>"+    TkPlusEq{} -> "+="+    TkMinusEq{} -> "-="+    TkStarEq{} -> "*="+    TkAtEq{} -> "@="+    TkAt{} -> "@"+    TkSlashEq{} -> "/="+    TkPercentEq{} -> "%="+    TkAmpersandEq{} -> "&="+    TkPipeEq{} -> "|="+    TkCaretEq{} -> "^="+    TkAmpersand{} -> "&"+    TkPipe{} -> "|"+    TkCaret{} -> "^"+    TkShiftLeftEq{} -> "<<="+    TkShiftRightEq{} -> ">>="+    TkDoubleStarEq{} -> "**="+    TkDoubleSlashEq{} -> "//="++whitespaceTokens :: Whitespace -> [PyToken ()]+whitespaceTokens Space = [TkSpace ()]+whitespaceTokens Tab = [TkTab ()]+whitespaceTokens (Continued nl ws) = TkContinued nl () : (ws >>= whitespaceTokens)+whitespaceTokens (Newline nl) = [TkNewline nl ()]+whitespaceTokens (Comment cmt) = commentTokens cmt++renderWhitespace :: Whitespace -> RenderOutput ()+renderWhitespace = traverse_ singleton . whitespaceTokens++renderNewline :: Newline -> PyToken ()+renderNewline nl = TkNewline nl ()++renderComma :: Comma -> RenderOutput ()+renderComma (MkComma ws) = do+  singleton $ TkComma ()+  traverse_ renderWhitespace ws++renderAt :: At -> RenderOutput ()+renderAt (MkAt ws) = do+  singleton $ TkAt ()+  traverse_ renderWhitespace ws++renderCommaSep :: (a -> RenderOutput ()) -> CommaSep a -> RenderOutput ()+renderCommaSep _ CommaSepNone = pure ()+renderCommaSep f (CommaSepOne a) = f a+renderCommaSep f (CommaSepMany a c cs) = do+  f a+  renderComma c+  renderCommaSep f cs++renderCommaSep1 :: (a -> RenderOutput ()) -> CommaSep1 a -> RenderOutput ()+renderCommaSep1 f (CommaSepOne1 a) = f a+renderCommaSep1 f (CommaSepMany1 a comma c) = do+  f a+  renderComma comma+  renderCommaSep1 f c++renderCommaSep1' :: (a -> RenderOutput ()) -> CommaSep1' a -> RenderOutput ()+renderCommaSep1' f (CommaSepOne1' a b) = do+  f a+  traverse_+    renderComma+    b+renderCommaSep1' f (CommaSepMany1' a comma c) = do+  f a+  renderComma comma+  renderCommaSep1' f c++renderIdent :: Ident v a -> RenderOutput ()+renderIdent (MkIdent _ a b) = do+  singleton $ TkIdent a ()+  traverse_ renderWhitespace b++parensTernaryLambda :: (Expr v a -> RenderOutput ()) -> Expr v a -> RenderOutput ()+parensTernaryLambda _ e@Ternary{} = parensDistTWS renderExpr e+parensTernaryLambda _ e@Lambda{} = parensDistTWS renderExpr e+parensTernaryLambda f e = f e++renderCompFor :: CompFor v a -> RenderOutput ()+renderCompFor (CompFor _ ws1 ex1 ws2 ex2) = do+  singleton $ TkFor ()+  traverse_ renderWhitespace ws1+  (case ex1 of+     Not{} -> parensDistTWS renderExpr ex1+     _ -> parensGenerator ex1)+  singleton $ TkIn ()+  traverse_ renderWhitespace ws2+  parensTernaryLambda parensTupleGenerator ex2++renderCompIf :: CompIf v a -> RenderOutput ()+renderCompIf (CompIf _ ws ex) = do+  singleton $ TkIf ()+  traverse_ renderWhitespace ws+  parensTernaryLambda parensTupleGenerator ex++renderComprehension+  :: (e v a -> RenderOutput ())+  -> Comprehension e v a+  -> RenderOutput ()+renderComprehension f (Comprehension _ expr cf cs) = do+  f expr+  renderCompFor cf+  traverse_ (bitraverse_ renderCompFor renderCompIf) cs++renderDictItem :: DictItem v a -> RenderOutput ()+renderDictItem (DictItem _ a b c) = do+  parensTupleGenerator a+  renderColon b+  parensTupleGenerator c+renderDictItem (DictUnpack _ a b) = do+  singleton $ TkDoubleStar ()+  traverse_ renderWhitespace a+  case b of+    BinOp _ _ BoolAnd{} _ -> parensDistTWS renderExpr b+    BinOp _ _ BoolOr{} _ -> parensDistTWS renderExpr b+    BinOp _ _ op _ | isComparison op -> parensDistTWS renderExpr b+    Not{} -> parensDistTWS renderExpr b+    _ -> parensTernaryLambda parensTupleGenerator b++renderStringLiteral :: StringLiteral a -> RenderOutput ()+renderStringLiteral (StringLiteral _ a b c d e) = do+  singleton $ TkString a b c d ()+  traverse_ renderWhitespace e+renderStringLiteral (BytesLiteral _ a b c d e) = do+  singleton $ TkBytes a b c d ()+  traverse_ renderWhitespace e+renderStringLiteral (RawStringLiteral _ a b c d e) = do+  singleton $ TkRawString a b c d ()+  traverse_ renderWhitespace e+renderStringLiteral (RawBytesLiteral _ a b c d e) = do+  singleton $ TkRawBytes a b c d ()+  traverse_ renderWhitespace e++renderSubscript :: Subscript v a -> RenderOutput ()+renderSubscript (SubscriptExpr a) =+  case a of+    Await{} -> parensDistTWS renderExpr a+    _ -> parensTupleGenerator a+renderSubscript (SubscriptSlice a b c d) = do+  traverse_ parensTupleGenerator a+  renderColon b+  traverse_ parensTupleGenerator c+  traverse_+    (bitraverse_+      renderColon+      (traverse_ parensTupleGenerator))+    d++renderYield :: (Expr v a -> RenderOutput ()) -> Expr v a -> RenderOutput ()+renderYield _ (Yield _ a b) = do+  singleton $ TkYield ()+  traverse_ renderWhitespace a+  renderCommaSep parensTupleGenerator b+renderYield _ (YieldFrom _ a b c) = do+  singleton $ TkYield ()+  traverse_ renderWhitespace a+  singleton $ TkFrom ()+  traverse_ renderWhitespace b+  parensTupleGenerator c+renderYield re e = re e++renderUnpackTarget :: Expr v a -> RenderOutput ()+renderUnpackTarget e =+  case e of+    BinOp _ _ BoolAnd{} _ -> parensDistTWS renderExpr e+    BinOp _ _ BoolOr{} _ -> parensDistTWS renderExpr e+    BinOp _ _ op _ | isComparison op -> parensDistTWS renderExpr e+    Not{} -> parensDistTWS renderExpr e+    _ -> parensTernaryLambda parensTupleGenerator e++renderNestedParens+  :: RenderOutput ()+  -> [([Whitespace], [Whitespace])]+  -> RenderOutput ()+renderNestedParens =+  foldr+    (\(ws1, ws2) y -> do+        singleton $ TkLeftParen ()+        traverse_ renderWhitespace ws1+        y+        singleton $ TkRightParen ()+        traverse_ renderWhitespace ws2)++renderTupleItems+  :: CommaSep1' (TupleItem v a)+  -> RenderOutput ()+renderTupleItems (CommaSepOne1' a Nothing) =+  case a of+    TupleItem _ b -> parensTupleGenerator b+    TupleUnpack _ b c d ->+      renderNestedParens+        (do+            singleton $ TkStar ()+            traverse_ renderWhitespace c+            renderUnpackTarget d)+        b+renderTupleItems (CommaSepOne1' a (Just comma)) = do+  (case a of+     TupleItem _ b -> parensTupleGenerator b+     TupleUnpack _ [] b c ->+       parens $ do+         singleton $ TkStar ()+         traverse_ renderWhitespace b+         renderUnpackTarget c+     TupleUnpack _ b c d ->+       renderNestedParens+         (do+             singleton $ TkStar ()+             traverse_ renderWhitespace c+             renderUnpackTarget d)+         b)+  renderComma comma+renderTupleItems (CommaSepMany1' a comma rest) = do+  (case a of+    TupleItem _ b -> parensTupleGenerator b+    TupleUnpack _ [] b c ->+      parens $ do+        singleton $ TkStar ()+        traverse_ renderWhitespace b+        renderUnpackTarget c+    TupleUnpack _ b c d ->+      renderNestedParens+        (do+            singleton $ TkStar ()+            traverse_ renderWhitespace c+            renderUnpackTarget d)+        b)+  renderComma comma+  renderTupleItems rest++renderSetItem :: SetItem v a -> RenderOutput ()+renderSetItem a =+  case a of+    SetItem _ b -> parensTupleGenerator b+    SetUnpack _ b c d ->+      renderNestedParens+        (do+            singleton $ TkStar ()+            traverse_ renderWhitespace c+            renderUnpackTarget d)+        b++renderSetItems :: CommaSep1' (SetItem v a) -> RenderOutput ()+renderSetItems (CommaSepOne1' a Nothing) =+  case a of+    SetItem _ b -> parensTupleGenerator b+    SetUnpack _ b c d ->+      renderNestedParens+        (do+            singleton $ TkStar ()+            traverse_ renderWhitespace c+            renderUnpackTarget d)+        b+renderSetItems (CommaSepOne1' a (Just comma)) = do+  (case a of+     SetItem _ b -> parensTupleGenerator b+     SetUnpack _ [] b c -> do+       singleton $ TkStar ()+       traverse_ renderWhitespace b+       renderUnpackTarget c+     SetUnpack _ b c d ->+       renderNestedParens+         (do+             singleton $ TkStar ()+             traverse_ renderWhitespace c+             renderUnpackTarget d)+         b)+  renderComma comma+renderSetItems (CommaSepMany1' a comma rest) = do+  (case a of+    SetItem _ b -> parensTupleGenerator b+    SetUnpack _ [] b c -> do+      singleton $ TkStar ()+      traverse_ renderWhitespace b+      renderUnpackTarget c+    SetUnpack _ b c d ->+      renderNestedParens+        (do+            singleton $ TkStar ()+            traverse_ renderWhitespace c+            renderUnpackTarget d)+        b)+  renderComma comma+  renderSetItems rest++renderListItems :: CommaSep1' (ListItem v a) -> RenderOutput ()+renderListItems (CommaSepOne1' a Nothing) =+  case a of+    ListItem _ b -> parensTupleGenerator b+    ListUnpack _ b c d ->+      renderNestedParens+        (do+            singleton $ TkStar ()+            traverse_ renderWhitespace c+            renderUnpackTarget d)+        b+renderListItems (CommaSepOne1' a (Just comma)) = do+  (case a of+     ListItem _ b -> parensTupleGenerator b+     ListUnpack _ [] b c -> do+       singleton $ TkStar ()+       traverse_ renderWhitespace b+       renderUnpackTarget c+     ListUnpack _ b c d ->+       renderNestedParens+         (do+             singleton $ TkStar ()+             traverse_ renderWhitespace c+             renderUnpackTarget d)+         b)+  renderComma comma+renderListItems (CommaSepMany1' a comma rest) = do+  (case a of+    ListItem _ b -> parensTupleGenerator b+    ListUnpack _ [] b c -> do+      singleton $ TkStar ()+      traverse_ renderWhitespace b+      renderUnpackTarget c+    ListUnpack _ b c d ->+      renderNestedParens+        (do+            singleton $ TkStar ()+            traverse_ renderWhitespace c+            renderUnpackTarget d)+        b)+  renderComma comma+  renderListItems rest++renderExpr :: Expr v a -> RenderOutput ()+renderExpr (Unit _ a b) = do+  singleton $ TkLeftParen ()+  traverse_ renderWhitespace a+  singleton $ TkRightParen ()+  traverse_ renderWhitespace b+renderExpr (Lambda _ a b c d) = do+  singleton $ TkLambda ()+  traverse_ renderWhitespace a+  renderParams b+  renderColon c+  parensTupleGenerator d+renderExpr e@Yield{} = parensDistTWS (renderYield parensTupleGenerator) e+renderExpr e@YieldFrom{} = parensDistTWS (renderYield parensTupleGenerator) e+renderExpr (Ternary _ a b c d e) = do+  (case a of+     Generator{} -> parensDistTWS renderExpr a+     _ -> parensTupleGenerator a)+  singleton $ TkIf ()+  traverse_ renderWhitespace b+  parensTernaryLambda parensTupleGenerator c+  singleton $ TkElse ()+  traverse_ renderWhitespace d+  parensTupleGenerator e+renderExpr (Subscript _ a b c d) = do+  (case a of+     BinOp{} -> parensDistTWS renderExpr a+     UnOp{} -> parensDistTWS renderExpr a+     Not{} -> parensDistTWS renderExpr a+     Ternary{} -> parensDistTWS renderExpr a+     Lambda{} -> parensDistTWS renderExpr a+     Await{} -> parensDistTWS renderExpr a+     _ -> parensTupleGenerator a)+  brackets $ do+    traverse_ renderWhitespace b+    renderCommaSep1' renderSubscript c+  traverse_ renderWhitespace d+renderExpr (Not _ ws e) = do+  singleton $ TkNot ()+  traverse_ renderWhitespace ws+  case e of+    BinOp _ _ BoolAnd{} _ -> parensDistTWS renderExpr e+    BinOp _ _ BoolOr{} _ -> parensDistTWS renderExpr e+    Ternary{} -> parensDistTWS renderExpr e+    Lambda{} -> parensDistTWS renderExpr e+    _ -> parensTupleGenerator e+renderExpr (Parens _ ws1 e ws2) = do+  parens $ do+    traverse_ renderWhitespace ws1+    renderYield renderExpr e+  traverse_ renderWhitespace ws2+renderExpr (Bool _ b ws) = do+  singleton $ if b then TkTrue () else TkFalse ()+  traverse_ renderWhitespace ws+renderExpr (UnOp _ op expr) = do+  renderUnOp op+  case expr of+    BinOp _ _ Exp{} _ -> parensTupleGenerator expr+    BinOp{} -> parensDistTWS renderExpr expr+    Deref _ Int{} _ _ -> parensDistTWS renderExpr expr+    Not{} -> parensDistTWS renderExpr expr+    Ternary{} -> parensDistTWS renderExpr expr+    Lambda{} -> parensDistTWS renderExpr expr+    _ -> parensTupleGenerator expr+renderExpr (String _ vs) =+  traverse_ renderStringLiteral $ correctAdjacentStrings vs+renderExpr (Int _ n ws) = do+  singleton $ TkInt (() <$ n)+  traverse_ renderWhitespace ws+renderExpr (Float _ n ws) = do+  singleton $ TkFloat (() <$ n)+  traverse_ renderWhitespace ws+renderExpr (Imag _ n ws) = do+  singleton $ TkImag (() <$ n)+  traverse_ renderWhitespace ws+renderExpr (Ident name) = renderIdent name+renderExpr (List _ ws1 exprs ws2) = do+  brackets $ do+    traverse_ renderWhitespace ws1+    traverse_ renderListItems exprs+  traverse_ renderWhitespace ws2+renderExpr (ListComp _ ws1 comp ws2) = do+  brackets $ do+    traverse_ renderWhitespace ws1+    renderComprehension+      (\e -> case e of+          Yield{} -> parensDistTWS renderExpr e+          YieldFrom{} -> parensDistTWS renderExpr e+          _ -> parensTupleGenerator e)+      comp+  traverse_ renderWhitespace ws2+renderExpr (Call _ expr ws args ws2) = do+  (case expr of+     UnOp{} -> parensDistTWS renderExpr expr+     BinOp{} -> parensDistTWS renderExpr expr+     Tuple{} -> parensDistTWS renderExpr expr+     Not{} -> parensDistTWS renderExpr expr+     Ternary{} -> parensDistTWS renderExpr expr+     Lambda{} -> parensDistTWS renderExpr expr+     _ -> parensGenerator expr)+  parens $ do+    traverse_ renderWhitespace ws+    traverse_ renderArgs args+  traverse_ renderWhitespace ws2+renderExpr (Deref _ expr ws name) = do+  (case expr of+     Int{} -> parensDistTWS renderExpr expr+     BinOp{} -> parensDistTWS renderExpr expr+     Tuple{} -> parensDistTWS renderExpr expr+     Not{} -> parensDistTWS renderExpr expr+     UnOp{} -> parensDistTWS renderExpr expr+     Ternary{} -> parensDistTWS renderExpr expr+     Lambda{} -> parensDistTWS renderExpr expr+     Await{} -> parensDistTWS renderExpr expr+     _ -> parensGenerator expr)+  singleton $ TkDot ()+  traverse_ renderWhitespace ws+  renderIdent name+renderExpr (None _ ws) = do+  singleton $ TkNone ()+  traverse_ renderWhitespace ws+renderExpr (Ellipsis _ ws) = do+  singleton $ TkEllipsis ()+  traverse_ renderWhitespace ws+renderExpr (BinOp _ e1 op e2) = do+  if shouldGroupLeft op e1+    then parensDistTWS renderExpr e1+    else parensTernaryLambda parensGenerator e1++  renderBinOp op++  if shouldGroupRight op e2+    then parensDistTWS renderExpr e2+    else parensTernaryLambda parensGenerator e2+renderExpr (Tuple _ a ws c) =+  renderTupleItems $+  case c of+    Nothing -> CommaSepOne1' a (Just ws)+    Just c' -> CommaSepMany1' a ws c'+renderExpr (DictComp _ ws1 comp ws2) = do+  braces $ do+    traverse_ renderWhitespace ws1+    renderComprehension renderDictItem comp+  traverse_ renderWhitespace ws2+renderExpr (Dict _ a b c) = do+  braces $ do+    traverse_ renderWhitespace a+    traverse_ (renderCommaSep1' renderDictItem) b+  traverse_ renderWhitespace c+renderExpr (SetComp _ ws1 comp ws2) = do+  braces $ do+    traverse_ renderWhitespace ws1+    renderComprehension renderSetItem comp+  traverse_ renderWhitespace ws2+renderExpr (Set _ a b c) = do+  braces $ do+    traverse_ renderWhitespace a+    renderSetItems b+  traverse_ renderWhitespace c+renderExpr (Generator _ a) =+  renderComprehension+    (\e -> case e of+        Yield{} -> parensDistTWS renderExpr e+        YieldFrom{} -> parensDistTWS renderExpr e+        _ -> parensTupleGenerator e)+    a+renderExpr (Await _ ws expr) = do+  singleton $ TkIdent "await" ()+  traverse_ renderWhitespace ws+  (case expr of+     UnOp{} -> parensDistTWS renderExpr expr+     BinOp{} -> parensDistTWS renderExpr expr+     Tuple{} -> parensDistTWS renderExpr expr+     Not{} -> parensDistTWS renderExpr expr+     Ternary{} -> parensDistTWS renderExpr expr+     Lambda{} -> parensDistTWS renderExpr expr+     Await{} -> parensDistTWS renderExpr expr+     _ -> parensGenerator expr)++renderModuleName :: ModuleName v a -> RenderOutput ()+renderModuleName (ModuleNameOne _ s) = renderIdent s+renderModuleName (ModuleNameMany _ n dot rest) = do+  renderIdent n+  renderDot dot+  renderModuleName rest++renderDot :: Dot -> RenderOutput ()+renderDot (MkDot ws) = do+  singleton $ TkDot ()+  traverse_ renderWhitespace ws++renderRelativeModuleName :: RelativeModuleName v a -> RenderOutput ()+renderRelativeModuleName (RelativeWithName ds mn) = do+  traverse_ renderDot ds+  renderModuleName mn+renderRelativeModuleName (Relative ds) =+  traverse_ renderDot ds++renderImportAs :: (e a -> RenderOutput ()) -> ImportAs e v a -> RenderOutput ()+renderImportAs f (ImportAs _ ea m) = do+  f ea+  traverse_+    (\(a, b) -> do+        singleton $ TkAs ()+        traverse_ renderWhitespace a+        renderIdent b)+    m++renderImportTargets :: ImportTargets v a -> RenderOutput ()+renderImportTargets (ImportAll _ ws) = do+  singleton $ TkStar ()+  traverse_ renderWhitespace ws+renderImportTargets (ImportSome _ ts) =+  renderCommaSep1 (renderImportAs renderIdent) ts+renderImportTargets (ImportSomeParens _ ws1 ts ws2) = do+  parens $ do+    traverse_ renderWhitespace ws1+    renderCommaSep1' (renderImportAs renderIdent) ts+  traverse_ renderWhitespace ws2++renderAugAssign :: AugAssign a -> RenderOutput ()+renderAugAssign aa = do+  singleton $ case _augAssignType aa of+    PlusEq -> TkPlusEq ()+    MinusEq -> TkMinusEq ()+    StarEq -> TkStarEq ()+    AtEq -> TkAtEq ()+    SlashEq -> TkSlashEq ()+    PercentEq -> TkPercentEq ()+    AmpersandEq -> TkAmpersandEq ()+    PipeEq -> TkPipeEq ()+    CaretEq -> TkCaretEq ()+    ShiftLeftEq -> TkShiftLeftEq ()+    ShiftRightEq -> TkShiftRightEq ()+    DoubleStarEq -> TkDoubleStarEq ()+    DoubleSlashEq -> TkDoubleSlashEq ()+  traverse_ renderWhitespace (_augAssignWhitespace aa)++renderSimpleStatement :: SimpleStatement v a -> RenderOutput ()+renderSimpleStatement (Assert _ b c d) = do+  singleton $ TkAssert ()+  traverse_ renderWhitespace b+  parensTupleGenerator c+  traverse_+    (\(a, b) -> do+        renderComma a+        parensTupleGenerator b)+    d+renderSimpleStatement (Raise _ ws x) = do+  singleton $ TkRaise ()+  traverse_ renderWhitespace ws+  traverse_+    (\(b, c) -> do+       parensTupleGenerator b+       traverse_+         (\(d, e) -> do+            singleton $ TkFrom ()+            traverse_ renderWhitespace d+            parensTupleGenerator e)+         c)+    x+renderSimpleStatement (Return _ ws expr) = do+  singleton $ TkReturn ()+  traverse_ renderWhitespace ws+  traverse_ parensGenerator expr+renderSimpleStatement (Expr _ expr) = renderYield parensGenerator expr+renderSimpleStatement (Assign _ lvalue rvalues) = do+  renderExpr lvalue+  traverse_+    (\(ws2, rvalue) -> do+       renderEquals ws2+       renderYield parensGenerator rvalue)+    rvalues+renderSimpleStatement (AugAssign _ lvalue as rvalue) = do+  renderExpr lvalue+  renderAugAssign as+  parensTupleGenerator rvalue+renderSimpleStatement (Pass _ ws) = do+  singleton $ TkPass ()+  traverse_ renderWhitespace ws+renderSimpleStatement (Continue _ ws) = do+  singleton $ TkContinue ()+  traverse_ renderWhitespace ws+renderSimpleStatement (Break _ ws) = do+  singleton $ TkBreak ()+  traverse_ renderWhitespace ws+renderSimpleStatement (Global _ ws ids) = do+  singleton $ TkGlobal ()+  traverse_ renderWhitespace ws+  renderCommaSep1 renderIdent ids+renderSimpleStatement (Nonlocal _ ws ids) = do+  singleton $ TkNonlocal ()+  traverse_ renderWhitespace ws+  renderCommaSep1 renderIdent ids+renderSimpleStatement (Del _ ws vals) = do+  singleton $ TkDel ()+  traverse_ renderWhitespace ws+  renderCommaSep1'+    (\a -> case a of+        BinOp{} -> parensDistTWS renderExpr a+        Not{} -> parensDistTWS renderExpr a+        Ternary{} -> parensDistTWS renderExpr a+        Lambda{} -> parensDistTWS renderExpr a+        _ -> parensTupleGenerator a)+    vals+renderSimpleStatement (Import _ ws ns) = do+  singleton $ TkImport ()+  traverse_ renderWhitespace ws+  renderCommaSep1 (renderImportAs renderModuleName) ns+renderSimpleStatement (From _ ws1 name ws3 ns) = do+  singleton $ TkFrom ()+  traverse_ renderWhitespace ws1+  renderRelativeModuleName name+  singleton $ TkImport ()+  traverse_ renderWhitespace ws3+  renderImportTargets ns++renderBlank :: Blank a -> RenderOutput ()+renderBlank (Blank _ a b) = do+  traverse_ renderWhitespace a+  traverse_ renderComment b++renderBlock :: Block v a -> RenderOutput ()+renderBlock (Block a b c) = do+  traverse_ (bitraverse_ renderBlank (singleton . renderNewline)) a+  (if null c then final else notFinal) $ renderStatement b+  traverseOf_+    (_init.traverse)+    (bitraverse_+      (bitraverse_ renderBlank (singleton . renderNewline))+      (notFinal . renderStatement))+    c+  traverseOf_+    _last+    (bitraverse_+      (bitraverse_ renderBlank (singleton . renderNewline))+      (final . renderStatement))+    c++renderSemicolon :: Semicolon a -> RenderOutput ()+renderSemicolon (MkSemicolon _ ws) = do+  singleton $ TkSemicolon ()+  traverse_ renderWhitespace ws++renderEquals :: Equals -> RenderOutput ()+renderEquals (MkEquals ws) = do+  singleton $ TkEq ()+  traverse_ renderWhitespace ws++renderColon :: Colon -> RenderOutput ()+renderColon (MkColon ws) = do+  singleton $ TkColon ()+  traverse_ renderWhitespace ws++renderSuite+  :: Suite v a+  -> RenderOutput ()+renderSuite (SuiteMany _ a b c d) = do+  renderColon a+  traverse_ renderComment b+  singleton (renderNewline c)+  renderBlock d+renderSuite (SuiteOne _ a b) = do+  renderColon a+  fin <- isFinal+  renderSmallStatement $ correctTrailingNewline fin b++renderDecorator :: Decorator v a -> RenderOutput ()+renderDecorator (Decorator _ a b c d e f) = do+  renderIndents a+  renderAt b+  renderExpr c+  traverse_ renderComment d+  singleton (renderNewline e)+  traverse_ (bitraverse_ renderBlank (singleton . renderNewline)) f++renderCompoundStatement :: CompoundStatement v a -> RenderOutput ()+renderCompoundStatement (Fundef _ decos idnt asyncWs ws1 name ws2 params ws3 mty s) = do+  traverse_ renderDecorator decos+  renderIndents idnt+  traverse_+    (\ws -> do+        singleton $ TkIdent "async" ()+        traverse_ renderWhitespace ws)+    asyncWs+  singleton (TkDef ())+  traverse_ renderWhitespace ws1+  renderIdent name+  parens $ do+    traverse_ renderWhitespace ws2+    renderParams params+  traverse_ renderWhitespace ws3+  traverse_+    (\(ws, ty) -> do+        singleton $ TkRightArrow ()+        traverse_ renderWhitespace ws+        parensTupleGenerator ty)+    mty+  final $ renderSuite s+renderCompoundStatement (If _ idnt ws1 expr s elifs body') = do+  renderIndents idnt+  singleton $ TkIf ()+  traverse_ renderWhitespace ws1+  parensTupleGenerator expr+  notFinal $ renderSuite s+  traverseOf_+    (_init.traverse)+    (\(idnt, ws4, ex, s) -> do+        renderIndents idnt+        singleton $ TkElif ()+        traverse_ renderWhitespace ws4+        parensTupleGenerator ex+        notFinal $ renderSuite s)+    elifs+  traverseOf_+    _last+    (\(idnt, ws4, ex, s) -> do+        renderIndents idnt+        singleton $ TkElif ()+        traverse_ renderWhitespace ws4+        parensTupleGenerator ex+        (if isNothing body' then final else notFinal) $ renderSuite s)+    elifs+  traverse_+    (\(idnt, ws4, s) -> do+        renderIndents idnt+        singleton $ TkElse ()+        traverse_ renderWhitespace ws4+        final $ renderSuite s)+    body'+renderCompoundStatement (While _ idnt ws1 expr s els) = do+  renderIndents idnt+  singleton $ TkWhile ()+  traverse_ renderWhitespace ws1+  parensTupleGenerator expr+  (if isNothing els then final else notFinal) $ renderSuite s+  traverse_+    (\(idnt, ws4, s) -> do+        renderIndents idnt+        singleton $ TkElse ()+        traverse_ renderWhitespace ws4+        final $ renderSuite s)+    els+renderCompoundStatement (TryExcept _ idnt a s e f g) = do+  renderIndents idnt+  singleton $ TkTry ()+  traverse_ renderWhitespace a+  notFinal $ renderSuite s+  traverse_+    (\(idnt, ws1, eas, s) -> do+       renderIndents idnt+       singleton $ TkExcept ()+       traverse_ renderWhitespace ws1+       traverse_ renderExceptAs eas+       notFinal $ renderSuite s)+    (NonEmpty.init e)+  (case NonEmpty.last e of+     (idnt, ws1, eas, s) -> do+       renderIndents idnt+       singleton $ TkExcept ()+       traverse_ renderWhitespace ws1+       traverse_ renderExceptAs eas+       (if isNothing f && isNothing g then final else notFinal) $ renderSuite s)+  traverse_+    (\(idnt, ws1, s) -> do+       renderIndents idnt+       singleton $ TkElse ()+       traverse_ renderWhitespace ws1+       (if isNothing g then final else notFinal) $ renderSuite s)+    f+  traverse_+    (\(idnt, ws1, s) -> do+       renderIndents idnt+       singleton $ TkFinally ()+       traverse_ renderWhitespace ws1+       final $ renderSuite s)+    g+renderCompoundStatement (TryFinally _ idnt a s idnt2 e s') = do+  renderIndents idnt+  singleton $ TkTry ()+  traverse_ renderWhitespace a+  notFinal $ renderSuite s+  renderIndents idnt2+  singleton $ TkFinally ()+  traverse_ renderWhitespace e+  final $ renderSuite s'+renderCompoundStatement (For _ idnt asyncWs a b c d s h) = do+  renderIndents idnt+  traverse_+    (\ws -> do+        singleton $ TkIdent "async" ()+        traverse_ renderWhitespace ws)+    asyncWs+  singleton $ TkFor ()+  traverse_ renderWhitespace a+  parensGenerator b+  singleton $ TkIn ()+  traverse_ renderWhitespace c+  renderCommaSep1' parensTupleGenerator d+  (if isNothing h then final else notFinal) $ renderSuite s+  traverse_+    (\(idnt, x, s) -> do+        renderIndents idnt+        singleton $ TkElse ()+        traverse_ renderWhitespace x+        final $ renderSuite s)+    h+renderCompoundStatement (ClassDef _ decos idnt a b c s) = do+  traverse_ renderDecorator decos+  renderIndents idnt+  singleton $ TkClass ()+  traverse_ renderWhitespace a+  renderIdent b+  traverse_+    (\(x, y, z) -> do+      parens $ do+        traverse_ renderWhitespace x+        traverse_ renderArgs y+      traverse_ renderWhitespace z)+    c+  final $ renderSuite s+renderCompoundStatement (With _ idnt asyncWs a b s) = do+  renderIndents idnt+  traverse_+    (\ws -> do+        singleton $ TkIdent "async" ()+        traverse_ renderWhitespace ws)+    asyncWs+  singleton $ TkWith ()+  traverse_ renderWhitespace a+  renderCommaSep1 renderWithItem b+  final $ renderSuite s++renderWithItem :: WithItem v a -> RenderOutput ()+renderWithItem (WithItem _ a b) = do+  parensTupleGenerator a+  traverse_+    (\(c, d) -> do+       singleton $ TkAs ()+       traverse_ renderWhitespace c+       parensTupleGenerator d)+    b++renderIndent :: Indent -> RenderOutput ()+renderIndent (MkIndent ws) = traverse_ renderWhitespace $ toList ws++renderSmallStatement :: SmallStatement v a -> RenderOutput ()+renderSmallStatement (MkSmallStatement s ss sc cmt nl) = do+  renderSimpleStatement s+  traverse_+    (\(b, c) -> do+       renderSemicolon b+       renderSimpleStatement c)+    ss+  traverse_ renderSemicolon sc+  traverse_ renderComment cmt+  traverse_ (singleton . renderNewline) nl++renderStatement :: Statement v a -> RenderOutput ()+renderStatement (CompoundStatement c) = renderCompoundStatement c+renderStatement (SmallStatement idnts s) = do+  renderIndents idnts+  fin <- isFinal+  renderSmallStatement $ correctTrailingNewline fin s++renderExceptAs :: ExceptAs v a -> RenderOutput ()+renderExceptAs (ExceptAs _ e f) = do+  parensTupleGenerator e+  traverse_+    (\(a, b) -> do+        singleton $ TkAs ()+        traverse_ renderWhitespace a+        renderIdent b)+    f++renderArgs :: CommaSep1' (Arg v a) -> RenderOutput ()+renderArgs (CommaSepOne1' a Nothing) = renderArg parensTuple a+renderArgs e = renderCommaSep1' (renderArg parensTupleGenerator) e++renderArg :: (Expr v a -> RenderOutput ()) -> Arg v a -> RenderOutput ()+renderArg re (PositionalArg _ expr) = re expr+renderArg _ (KeywordArg _ name ws2 expr) = do+  renderIdent name+  singleton $ TkEq ()+  traverse_ renderWhitespace ws2+  parensTupleGenerator expr+renderArg _ (StarArg _ ws expr) = do+  singleton $ TkStar ()+  traverse_ renderWhitespace ws+  parensTupleGenerator expr+renderArg _ (DoubleStarArg _ ws expr) = do+  singleton $ TkDoubleStar ()+  traverse_ renderWhitespace ws+  parensTupleGenerator expr++renderParams :: CommaSep (Param v a) -> RenderOutput ()+renderParams = renderCommaSep renderParam . correctParams++renderParam :: Param v a -> RenderOutput ()+renderParam (PositionalParam _ name mty) = do+  renderIdent name+  traverse_+    (\(c, ty) -> do+        renderColon c+        parensTupleGenerator ty)+    mty+renderParam (StarParam _ ws name mty) = do+  singleton $ TkStar ()+  traverse_ renderWhitespace ws+  renderIdent name+  traverse_+    (\(c, ty) -> do+        renderColon c+        parensTupleGenerator ty)+    mty+renderParam (UnnamedStarParam _ ws) = do+  singleton $ TkStar ()+  traverse_ renderWhitespace ws+renderParam (DoubleStarParam _ ws name mty) = do+  singleton $ TkDoubleStar ()+  traverse_ renderWhitespace ws+  renderIdent name+  traverse_+    (\(c, ty) -> do+        renderColon c+        parensTupleGenerator ty)+    mty+renderParam (KeywordParam _ name mty ws2 expr) = do+  renderIdent name+  traverse_+    (\(c, ty) -> do+        renderColon c+        parensTupleGenerator ty)+    mty+  singleton $ TkEq ()+  traverse_ renderWhitespace ws2+  parensTupleGenerator expr++renderUnOp :: UnOp a -> RenderOutput ()+renderUnOp (Negate _ ws) = do+  singleton $ TkMinus ()+  traverse_ renderWhitespace ws+renderUnOp (Positive _ ws) = do+  singleton $ TkPlus ()+  traverse_ renderWhitespace ws+renderUnOp (Complement _ ws) = do+  singleton $ TkTilde ()+  traverse_ renderWhitespace ws++renderBinOp :: BinOp a -> RenderOutput ()+renderBinOp (Is _ ws) = do+  singleton $ TkIs ()+  traverse_ renderWhitespace ws+renderBinOp (IsNot _ ws1 ws2) = do+  singleton $ TkIs ()+  traverse_ renderWhitespace ws1+  singleton $ TkNot ()+  traverse_ renderWhitespace ws2+renderBinOp (In _ ws) = do+  singleton $ TkIn ()+  traverse_ renderWhitespace ws+renderBinOp (NotIn _ ws1 ws2) = do+  singleton $ TkNot ()+  traverse_ renderWhitespace ws1+  singleton $ TkIn ()+  traverse_ renderWhitespace ws2+renderBinOp (Plus _ ws) = do+  singleton $ TkPlus ()+  traverse_ renderWhitespace ws+renderBinOp (Minus _ ws) = do+  singleton $ TkMinus ()+  traverse_ renderWhitespace ws+renderBinOp (Multiply _ ws) = do+  singleton $ TkStar ()+  traverse_ renderWhitespace ws+renderBinOp (At _ ws) = do+  singleton $ TkAt ()+  traverse_ renderWhitespace ws+renderBinOp (Divide _ ws) = do+  singleton $ TkSlash ()+  traverse_ renderWhitespace ws+renderBinOp (FloorDivide _ ws) = do+  singleton $ TkDoubleSlash ()+  traverse_ renderWhitespace ws+renderBinOp (Exp _ ws) = do+  singleton $ TkDoubleStar ()+  traverse_ renderWhitespace ws+renderBinOp (BoolAnd _ ws) = do+  singleton $ TkAnd ()+  traverse_ renderWhitespace ws+renderBinOp (BoolOr _ ws) = do+  singleton $ TkOr ()+  traverse_ renderWhitespace ws+renderBinOp (Eq _ ws) = do+  singleton $ TkDoubleEq ()+  traverse_ renderWhitespace ws+renderBinOp (Lt _ ws) = do+  singleton $ TkLt ()+  traverse_ renderWhitespace ws+renderBinOp (LtEq _ ws) = do+  singleton $ TkLte ()+  traverse_ renderWhitespace ws+renderBinOp (Gt _ ws) = do+  singleton $ TkGt ()+  traverse_ renderWhitespace ws+renderBinOp (GtEq _ ws) = do+  singleton $ TkGte ()+  traverse_ renderWhitespace ws+renderBinOp (NotEq _ ws) = do+  singleton $ TkBangEq ()+  traverse_ renderWhitespace ws+renderBinOp (Percent _ ws) = do+  singleton $ TkPercent ()+  traverse_ renderWhitespace ws+renderBinOp (BitOr _ ws) = do+  singleton $ TkPipe ()+  traverse_ renderWhitespace ws+renderBinOp (BitXor _ ws) = do+  singleton $ TkCaret ()+  traverse_ renderWhitespace ws+renderBinOp (BitAnd _ ws) = do+  singleton $ TkAmpersand ()+  traverse_ renderWhitespace ws+renderBinOp (ShiftLeft _ ws) = do+  singleton $ TkShiftLeft ()+  traverse_ renderWhitespace ws+renderBinOp (ShiftRight _ ws) = do+  singleton $ TkShiftRight ()+  traverse_ renderWhitespace ws++renderIndents :: Indents a -> RenderOutput ()+renderIndents (Indents is _) = traverse_ renderIndent is++renderModule :: Module v a -> RenderOutput ()+renderModule ModuleEmpty = pure ()+renderModule (ModuleBlankFinal a) = renderBlank a+renderModule (ModuleBlank a b c) = do+  renderBlank a+  singleton $ renderNewline b+  renderModule c+renderModule (ModuleStatement a b) = do+  renderStatement a+  renderModule b++-- | Render an entire Python module to 'Text'+showModule :: Module v a -> Text+showModule = showRenderOutput . renderModule++-- | Render a single Python statement to 'Text'+showStatement :: Statement v a -> Text+showStatement = showRenderOutput . renderStatement++-- | Render a single Python expression to 'Text'+showExpr :: Expr v a -> Text+showExpr = showRenderOutput . parensGenerator
+ src/Language/Python/Internal/Render/Correction.hs view
@@ -0,0 +1,326 @@+{-# language BangPatterns #-}+{-# language LambdaCase #-}++{-|+Module      : Language.Python.Internal.Render.Correction+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++There are configurations of the core syntax tree which won't print to valid Python+if we printed them naively. Many of these we catch in the+'Language.Python.Validation.Syntax' phase, because those mistakes correspond to+some Python syntax error. In other cases, the mistakes are more benign and have+a "resonable correction" which doesn't break the "print-parse idempotence" law.++This module is where such corrections are defined+-}++module Language.Python.Internal.Render.Correction where++import Control.Lens.Fold (hasn't)+import Control.Lens.Getter ((^.))+import Control.Lens.Plated (transform)+import Control.Lens.Setter ((.~), (<>~))+import Data.Function ((&))+import Data.List.NonEmpty (NonEmpty(..))+import Data.Semigroup ((<>))+import Data.Text (Text)++import qualified Data.List.NonEmpty as NonEmpty+import qualified Data.Text as Text++import Language.Python.Internal.Token+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace++-- | Trailing commas can only be present in a parameter list of entirely+-- positional arguments. This removes the bad trailing comma, and appends+-- the comma's trailing whitespace to the previous token+correctParams :: CommaSep (Param v a) -> CommaSep (Param v a)+correctParams CommaSepNone = CommaSepNone+correctParams (CommaSepOne a) = CommaSepOne a+correctParams (CommaSepMany a (MkComma b) c) =+  case c of+    CommaSepNone ->+      case a of+        PositionalParam{} -> CommaSepMany a (MkComma b) c+        _ -> CommaSepOne (a & trailingWhitespace <>~ b)+    _ -> CommaSepMany a (MkComma b) (correctParams c)++correctSpaces :: (PyToken () -> Text) -> [PyToken ()] -> [PyToken ()]+correctSpaces f =+  transform $+  \case+    a : b : rest+      | isIdentifierChar (Text.last $ f a)+      , isIdentifierChar (Text.head $ f b)+      -> a : TkSpace () : b : rest+    a@(TkFloat (FloatLiteralFull _ _ Nothing)) : b : rest+      | isIdentifierChar (Text.head $ f b) -> a : TkSpace () : b : rest+    a -> a++correctNewlines :: [PyToken ()] -> [PyToken ()]+correctNewlines =+  transform $+  \case+    TkNewline CR () : TkNewline LF () : rest ->+      TkNewline CRLF () : TkNewline LF () : rest+    TkContinued CR () : TkNewline LF () : rest ->+      TkContinued CRLF () : TkNewline LF () : rest+    a -> a++-- |+-- Two non-typed single-quoted strings cannot be lexically+-- adjacent, because this would be a parse error+--+-- eg. '''' or """"+--+-- we correct for this by inserting a single space where required+-- '' '' or "" ""+correctAdjacentStrings :: NonEmpty (StringLiteral a) -> NonEmpty (StringLiteral a)+correctAdjacentStrings (a :| []) = a :| []+correctAdjacentStrings (a:|b:cs) =+  if+    _stringLiteralQuoteType a == _stringLiteralQuoteType b &&+    _stringLiteralStringType a == _stringLiteralStringType b &&+    null (a ^. trailingWhitespace) &&+    not (hasPrefix b)+  then+    NonEmpty.cons (a & trailingWhitespace .~ [Space]) (correctAdjacentStrings $ b :| cs)+  else+    NonEmpty.cons a $ correctAdjacentStrings (b :| cs)++quoteChar :: QuoteType -> PyChar+quoteChar qt =+  case qt of+    SingleQuote -> Char_esc_singlequote+    DoubleQuote -> Char_esc_doublequote++quote :: QuoteType -> Char+quote qt =+  case qt of+    DoubleQuote -> '\"'+    SingleQuote -> '\''++-- | When a backslash character, precedes an escape sequence it needs to be escaped+-- so that it doesn't interfere with the backslash that begins the escape sequence.+--+-- For example:+--+-- @['Char_lit' \'\\\\\', Char_esc_n]@ would naively render to \'\\\\n\', which+-- would parse to @['Char_esc_bslash', 'Char_lit' \'n\']@, breaking the+-- @parse . print@ identity+correctBackslashEscapes :: [PyChar] -> [PyChar]+correctBackslashEscapes [] = []+correctBackslashEscapes [x] = [x]+correctBackslashEscapes (x:y:ys) =+  case x of+    Char_lit '\\'+      -- if the next character is an escape sequence, then the current backslash+      -- must be escaped+      | isEscape y -> Char_esc_bslash : y : correctBackslashEscapes ys+      | Char_lit c <- y ->+        case c of+          '\\' -> Char_esc_bslash : correctBackslashEscapes ys+          '\'' -> Char_esc_bslash : correctBackslashEscapes ys+          '\"' -> Char_esc_bslash : correctBackslashEscapes ys+          -- if we print out ['\', 'u'] then the parser will think it's beginning a+          -- unicode point+          'u' -> Char_esc_bslash : y : correctBackslashEscapes ys+          'U' -> Char_esc_bslash : y : correctBackslashEscapes ys+          -- same for 'x' and hex values+          'x' -> Char_esc_bslash : y : correctBackslashEscapes ys+          _ -> x : correctBackslashEscapes (y : ys)+    _ -> x : correctBackslashEscapes (y : ys)++correctBackslashes :: [PyChar] -> [PyChar]+correctBackslashes [] = []+correctBackslashes [x] =+  case x of+    Char_lit '\\' -> [Char_esc_bslash]+    _ -> [x]+correctBackslashes (x:y:ys) =+  case x of+    Char_lit '\\'+      -- if the next character is an escape sequence, then the current backslash+      -- must be escaped+      | Char_esc_bslash <- y -> Char_esc_bslash : y : correctBackslashes ys+    _ -> x : correctBackslashes (y : ys)++-- | @(as, bs) = span p xs@+-- @bs@ is the longest suffix that satisfies the predicate, and @as@ is the+-- prefix up to that point+--+-- It's like the reverse of 'span'+naps :: (a -> Maybe b) -> [a] -> ([a], [b])+naps p = go (,) (,)+  where+    go _ r [] = r [] []+    go l r (x:xs) =+      go+        (\res res' -> l (x:res) res')+        (\res res' ->+           case p x of+             Just x' -> r res (x':res')+             Nothing -> l (x:res) res')+        xs++-- | Sometimes strings need to be corrected when certain characters follow a literal+-- backslash. For example, a literal backslash followed by an escape sequence means+-- that the literal backslash actually needs to be escaped, so that it doesn't get+-- 'combined' with the backslash in the escape sequence.+correctBackslashEscapesRaw :: [PyChar] -> [PyChar]+correctBackslashEscapesRaw [] = []+correctBackslashEscapesRaw [x] = [x]+correctBackslashEscapesRaw(x:y:ys) =+  case x of+    Char_lit '\\' ->+      case y of+        Char_esc_doublequote -> Char_esc_bslash : y : correctBackslashEscapesRaw ys+        Char_esc_singlequote -> Char_esc_bslash : y : correctBackslashEscapesRaw ys+        Char_esc_bslash -> Char_esc_bslash : correctBackslashEscapesRaw (Char_lit '\\' : ys)+        _ -> x : correctBackslashEscapesRaw (y : ys)+    _ -> x : correctBackslashEscapesRaw (y : ys)++-- | It turns out that raw strings can only ever be constructed with an even number+-- of trailing backslash characters. This functon corrects raw strings with an+-- odd number of trailing backslash characters+correctBackslashesRaw :: [PyChar] -> [PyChar]+correctBackslashesRaw ps =+  let+    (as, bs) =+      naps+        (\a ->+           case a of+             Char_lit '\\' -> Just a+             Char_esc_bslash -> Just a+             _ -> Nothing)+        ps+  in+    if even (numSlashes bs)+    then ps+    else+      as <> (Char_lit '\\' : bs)+  where+    numSlashes :: [PyChar] -> Int+    numSlashes [] = 0+    numSlashes (Char_lit '\\' : xs) = 1 + numSlashes xs+    numSlashes (Char_esc_bslash : xs) = 2 + numSlashes xs+    numSlashes _ = undefined++-- | Every quote in a string of a particular quote type should be escaped+correctQuotes :: QuoteType -> [PyChar] -> [PyChar]+correctQuotes qt =+  fmap+    (case qt of+       DoubleQuote -> \case; Char_lit '"' -> Char_esc_doublequote; c -> c+       SingleQuote -> \case; Char_lit '\'' -> Char_esc_singlequote; c -> c)++-- | Every quote in short raw string that isn't preceded by+-- a backslash should be escaped+correctQuotesRaw :: QuoteType -> [PyChar] -> [PyChar]+correctQuotesRaw _ [] = []+correctQuotesRaw qt [x] =+  case x of+    Char_lit c | quote qt == c -> [quoteChar qt]+    _ -> [x]+correctQuotesRaw qt (x:y:ys) =+  case x of+    Char_lit c | q == c -> go (qc:y:ys)+    _ -> go (x:y:ys)+  where+    qc = quoteChar qt+    q = quote qt++    go [] = []+    go [x] = [x]+    go (x:y:ys) =+      case x of+        Char_lit '\\' -> x : go (y:ys)+        _ ->+          case y of+            Char_lit c | q == c -> x : go (qc:ys)+            _ -> x : go (y:ys)++-- | Every third literal quote at the beginning of a long (non-raw) string should+-- be escaped+correctInitialQuotes :: QuoteType -> [PyChar] -> [PyChar]+correctInitialQuotes qt = go (0::Int)+  where+    qc = quoteChar qt+    q = quote qt++    go !_ [] = []+    go !n (c:cs) =+      if c == Char_lit q+      then+        if n == 2+        then qc : go (n+1 `mod` 3) cs+        else c : go (n+1 `mod` 3) cs+      else c : cs++-- | Literal quotes at the beginning and end of a long raw string should be escaped+correctInitialFinalQuotesLongRaw :: QuoteType -> [PyChar] -> [PyChar]+correctInitialFinalQuotesLongRaw qt = correctFinalQuotes . correctInitialQuotes qt+  where+    qc = quoteChar qt+    q = quote qt++    -- | Literal quotes at the end of a long raw string should be escaped+    correctFinalQuotes :: [PyChar] -> [PyChar]+    correctFinalQuotes = snd . go+      where+        go [] = (True, [])+        go (c:cs) =+          if c /= Char_lit '\\'+          then+            case go cs of+              (b, cs') ->+                if b && c == Char_lit q+                then (True, qc : cs')+                else (False, c : cs')+          else+            let+              (ds, es) = span (== Char_lit '\\') cs+            in+              case es of+                [] -> (False, c : ds)+                e':es' ->+                  case go es' of+                    (_, es'') -> (False, c : ds <> (e' : es''))++-- | Literal quotes at the beginning and end of a long (non-raw) string should be escaped+correctInitialFinalQuotesLong :: QuoteType -> [PyChar] -> [PyChar]+correctInitialFinalQuotesLong qt = correctFinalQuotes . correctInitialQuotes qt+  where+    qc = quoteChar qt+    q = quote qt++    -- | Literal quotes at the end of a long (non-raw) string should be escaped+    correctFinalQuotes :: [PyChar] -> [PyChar]+    correctFinalQuotes = snd . go+      where+        go [] = (True, [])+        go (c:cs) =+          case go cs of+            (b, cs') ->+              if b && c == Char_lit q+              then (True, qc : cs')+              else (False, c : cs')++-- | It's possible that successive statements have no newlines in between+-- them. This would cause them to be displayed on the same line. In every line where+-- this would be the case, we explicitly insert a line-feed character.+correctTrailingNewline :: HasTrailingNewline s => Bool -> s v a -> s v a+correctTrailingNewline False s =+  if hasn't trailingNewline s+  then setTrailingNewline s LF+  else s+correctTrailingNewline True s = s
+ src/Language/Python/Internal/Syntax/IR.hs view
@@ -0,0 +1,1005 @@+{-# language DataKinds #-}+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language FunctionalDependencies, MultiParamTypeClasses #-}+{-# language LambdaCase #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Internal.Syntax.IR+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++This module only exists as our current best solution to decoupling parts of the+concrete syntax from abstract syntax. You won't need to care about its existence+and hopefully it will be deleted soon.++-}++module Language.Python.Internal.Syntax.IR where++import Control.Lens.Fold (foldMapOf, folded)+import Control.Lens.Getter ((^.))+import Control.Lens.Lens (Lens', lens)+import Control.Lens.Prism (Prism')+import Control.Lens.Review ((#))+import Control.Lens.Setter ((.~), over, mapped)+import Control.Lens.TH (makeLenses)+import Control.Lens.Traversal (traverseOf)+import Control.Lens.Tuple (_1, _2, _3)+import Data.Bifoldable (bifoldMap)+import Data.Bifunctor (bimap)+import Data.Bitraversable (bitraverse)+import Data.Function ((&))+import Data.List.NonEmpty (NonEmpty)+import Data.Monoid ((<>))+import Data.Validation (Validation(..))++import Language.Python.Syntax.AugAssign+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Comment+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Import+import Language.Python.Syntax.ModuleNames+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Operator.Binary+import Language.Python.Syntax.Operator.Unary+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace++import qualified Language.Python.Syntax.Module as Syntax+import qualified Language.Python.Syntax.Expr as Syntax+import qualified Language.Python.Syntax.Statement as Syntax++class AsIRError s a | s -> a where+  _InvalidUnpacking :: Prism' s a++data IRError a+  -- | Unpacking ( @*value@ ) was used in an invalid position+  = InvalidUnpacking a+  deriving (Eq, Show)++fromIRError :: AsIRError s a => IRError a -> s+fromIRError (InvalidUnpacking a) = _InvalidUnpacking # a++data SmallStatement a+  = MkSmallStatement+      (SimpleStatement a)+      [(Semicolon a, SimpleStatement a)]+      (Maybe (Semicolon a))+      (Maybe (Comment a))+      (Maybe Newline)+  deriving (Eq, Show, Functor, Foldable, Traversable)++data Statement a+  = SmallStatement (Indents a) (SmallStatement a)+  | CompoundStatement (CompoundStatement a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++data CompoundStatement a+  = Fundef+  { _csAnn :: a+  , _unsafeCsFundefDecorators :: [Decorator a]+  , _csIndents :: Indents a+  , _unsafeCsFundefAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \<spaces\>]@+  , _unsafeCsFundefDef :: NonEmpty Whitespace -- ^ @\'def\' \<spaces\>@+  , _unsafeCsFundefName :: Ident '[] a -- ^ @\<ident\>@+  , _unsafeCsFundefLeftParen :: [Whitespace] -- ^ @\'(\' \<spaces\>@+  , _unsafeCsFundefParameters :: CommaSep (Param a) -- ^ @\<parameters\>@+  , _unsafeCsFundefRightParen :: [Whitespace] -- ^ @\')\' \<spaces\>@+  , _unsafeCsFundefReturnType :: Maybe ([Whitespace], Expr a) -- ^ @[\'->\' \<spaces\> \<expr\>]@+  , _unsafeCsFundefBody :: Suite a -- ^ @\<suite\>@+  }+  | If+  { _csAnn :: a+  , _csIndents :: Indents a+  , _unsafeCsIfIf :: [Whitespace] -- ^ @\'if\' \<spaces\>@+  , _unsafeCsIfCond :: Expr a -- ^ @\<expr\>@+  , _unsafeCsIfBody :: Suite a -- ^ @\<suite\>@+  , _unsafeCsIfElifs :: [(Indents a, [Whitespace], Expr a, Suite a)] -- ^ @(\'elif\' \<spaces\> \<expr\> \<suite\>)*@+  , _unsafeCsIfElse :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  }+  | While+  { _csAnn :: a+  , _csIndents :: Indents a+  , _unsafeCsWhileWhile :: [Whitespace] -- ^ @\'while\' \<spaces\>@+  , _unsafeCsWhileCond :: Expr a -- ^ @\<expr\>@+  , _unsafeCsWhileBody :: Suite a -- ^ @\<suite\>@+  , _unsafeCsWhileElse+    :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  }+  | TryExcept+  { _csAnn :: a+  , _csIndents :: Indents a+  , _unsafeCsTryExceptTry :: [Whitespace] -- ^ @\'try\' \<spaces\>@+  , _unsafeCsTryExceptBody :: Suite a -- ^ @\<suite\>@+  , _unsafeCsTryExceptExcepts :: NonEmpty (Indents a, [Whitespace], Maybe (ExceptAs a), Suite a) -- ^ @(\'except\' \<spaces\> \<except_as\> \<suite\>)+@+  , _unsafeCsTryExceptElse :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  , _unsafeCsTryExceptFinally :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'finally\' \<spaces\> \<suite\>]@+  }+  | TryFinally+  { _csAnn :: a+  , _csIndents :: Indents a+  , _unsafeCsTryFinallyTry :: [Whitespace] -- ^ @\'try\' \<spaces\>@+  , _unsafeCsTryFinallyTryBody :: Suite a -- ^ @\<suite\>@+  , _unsafeCsTryFinallyFinallyIndents :: Indents a+  , _unsafeCsTryFinallyFinally :: [Whitespace] -- ^ @\'finally\' \<spaces\>@+  , _unsafeCsTryFinallyFinallyBody :: Suite a -- ^ @\<suite\>@+  }+  | For+  { _csAnn :: a+  , _csIndents :: Indents a+  , _unsafeCsForAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \<spaces\>]@+  , _unsafeCsForFor :: [Whitespace] -- ^ @\'for\' \<spaces\>@+  , _unsafeCsForBinder :: Expr a -- ^ @\<expr\>@+  , _unsafeCsForIn :: [Whitespace] -- ^ @\'in\' \<spaces\>@+  , _unsafeCsForCollection :: CommaSep1' (Expr a) -- ^ @\<exprs\>@+  , _unsafeCsForBody :: Suite a -- ^ @\<suite\>@+  , _unsafeCsForElse :: Maybe (Indents a, [Whitespace], Suite a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  }+  | ClassDef+  { _csAnn :: a+  , _unsafeCsClassDefDecorators :: [Decorator a]+  , _csIndents :: Indents a+  , _unsafeCsClassDefClass :: NonEmpty Whitespace -- ^ @\'class\' \<spaces\>@+  , _unsafeCsClassDefName :: Ident '[] a -- ^ @\<ident\>@+  , _unsafeCsClassDefArguments :: Maybe ([Whitespace], Maybe (CommaSep1' (Arg a)), [Whitespace]) -- ^ @[\'(\' \<spaces\> [\<args\>] \')\' \<spaces\>]@+  , _unsafeCsClassDefBody :: Suite a -- ^ @\<suite\>@+  }+  | With+  { _csAnn :: a+  , _csIndents :: Indents a+  , _unsafeCsWithAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \<spaces\>]@+  , _unsafeCsWithWith :: [Whitespace] -- ^ @\'with\' \<spaces\>@+  , _unsafeCsWithItems :: CommaSep1 (WithItem a) -- ^ @\<with_items\>@+  , _unsafeCsWithBody :: Suite a -- ^ @\<suite\>@+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++data SimpleStatement a+  = Return a [Whitespace] (Maybe (Expr a))+  | Expr a (Expr a)+  | Assign a (Expr a) (NonEmpty (Equals, Expr a))+  | AugAssign a (Expr a) (AugAssign a) (Expr a)+  | Pass a [Whitespace]+  | Break a [Whitespace]+  | Continue a [Whitespace]+  | Global a (NonEmpty Whitespace) (CommaSep1 (Ident '[] a))+  | Nonlocal a (NonEmpty Whitespace) (CommaSep1 (Ident '[] a))+  | Del a [Whitespace] (CommaSep1' (Expr a))+  | Import+      a+      (NonEmpty Whitespace)+      (CommaSep1 (ImportAs (ModuleName '[]) '[] a))+  | From+      a+      [Whitespace]+      (RelativeModuleName '[] a)+      [Whitespace]+      (ImportTargets '[] a)+  | Raise a+      [Whitespace]+      (Maybe (Expr a, Maybe ([Whitespace], Expr a)))+  | Assert a+      [Whitespace]+      (Expr a)+      (Maybe (Comma, Expr a))+  deriving (Eq, Show, Functor, Foldable, Traversable)++data Param a+  = PositionalParam+  { _paramAnn :: a+  , _paramName :: Ident '[] a+  , _paramType :: Maybe (Colon, Expr a)+  }+  | KeywordParam+  { _paramAnn :: a+  , _paramName :: Ident '[] a+  -- ':' spaces <expr>+  , _paramType :: Maybe (Colon, Expr a)+  -- = spaces+  , _unsafeKeywordParamWhitespaceRight :: [Whitespace]+  , _unsafeKeywordParamExpr :: Expr a+  }+  | StarParam+  { _paramAnn :: a+  -- '*' spaces+  , _unsafeStarParamWhitespace :: [Whitespace]+  , _unsafeStarParamName :: Ident '[] a+  , _paramType :: Maybe (Colon, Expr a)+  }+  | UnnamedStarParam+  { _paramAnn :: a+  -- '*' spaces+  , _unsafeUnnamedStarParamWhitespace :: [Whitespace]+  }+  | DoubleStarParam+  { _paramAnn :: a+  -- '**' spaces+  , _unsafeDoubleStarParamWhitespace :: [Whitespace]+  , _paramName :: Ident '[] a+  , _paramType :: Maybe (Colon, Expr a)+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++data CompIf a+  = CompIf a [Whitespace] (Expr a) -- ^ 'if' <any_spaces> <expr>+  deriving (Eq, Show, Functor, Foldable, Traversable)++data CompFor a+  = CompFor a [Whitespace] (Expr a) [Whitespace] (Expr a) -- ^ 'for' <any_spaces> <targets> 'in' <any_spaces> <expr>+  deriving (Eq, Show, Functor, Foldable, Traversable)++data Comprehension e a+  = Comprehension a (e a) (CompFor a) [Either (CompFor a) (CompIf a)] -- ^ <expr> <comp_for> (comp_for | comp_if)*+  deriving (Eq, Show)++instance Functor e => Functor (Comprehension e) where+  fmap f (Comprehension a b c d) =+    Comprehension (f a) (fmap f b) (fmap f c) (fmap (bimap (fmap f) (fmap f)) d)++instance Foldable e => Foldable (Comprehension e) where+  foldMap f (Comprehension a b c d) =+    f a <> foldMap f b <> foldMap f c <> foldMap (bifoldMap (foldMap f) (foldMap f)) d++instance Traversable e => Traversable (Comprehension e) where+  traverse f (Comprehension a b c d) =+    Comprehension <$>+    f a <*>+    traverse f b <*>+    traverse f c <*>+    traverse (bitraverse (traverse f) (traverse f)) d++data Subscript a+  = SubscriptExpr (Expr a)+  | SubscriptSlice+      -- [expr]+      (Maybe (Expr a))+      -- ':' <spaces>+      Colon+      -- [expr]+      (Maybe (Expr a))+      -- [':' [expr]]+      (Maybe (Colon, Maybe (Expr a)))+  deriving (Eq, Show, Functor, Foldable, Traversable)++data DictItem a+  = DictItem+  { _dictItemAnn :: a+  , _unsafeDictItemKey :: Expr a+  , _unsafeDictItemColon :: Colon+  , _unsafeDictItemvalue :: Expr a+  }+  | DictUnpack+  { _dictItemAnn :: a+  , _unsafeDictItemUnpackWhitespace :: [Whitespace]+  , _unsafeDictItemUnpackValue :: Expr a+  } deriving (Eq, Show, Functor, Foldable, Traversable)++data Arg a+  = PositionalArg+  { _argAnn :: a+  , _argExpr :: Expr a+  }+  | KeywordArg+  { _argAnn :: a+  , _unsafeKeywordArgName :: Ident '[] a+  , _unsafeKeywordArgWhitespaceRight :: [Whitespace]+  , _argExpr :: Expr a+  }+  | StarArg+  { _argAnn :: a+  , _unsafeStarArgWhitespace :: [Whitespace]+  , _argExpr :: Expr a+  }+  | DoubleStarArg+  { _argAnn :: a+  , _unsafeDoubleStarArgWhitespace :: [Whitespace]+  , _argExpr :: Expr a+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++data Expr a+  = StarExpr+  { _unsafeExprAnn :: a+  , _unsafeStarExprWhitespace :: [Whitespace]+  , _unsafeStarExprValue :: Expr a+  }+  | Unit+  { _unsafeExprAnn :: a+  , _unsafeUnitWhitespaceInner :: [Whitespace]+  , _unsafeUnitWhitespaceRight :: [Whitespace]+  }+  | Lambda+  { _unsafeExprAnn :: a+  , _unsafeLambdaWhitespace :: [Whitespace]+  , _unsafeLambdaArgs :: CommaSep (Param a)+  , _unsafeLambdaColon :: Colon+  , _unsafeLambdaBody :: Expr a+  }+  | Yield+  { _unsafeExprAnn :: a+  , _unsafeYieldWhitespace :: [Whitespace]+  , _unsafeYieldValue :: CommaSep (Expr a)+  }+  | YieldFrom+  { _unsafeExprAnn :: a+  , _unsafeYieldWhitespace :: [Whitespace]+  , _unsafeFromWhitespace :: [Whitespace]+  , _unsafeYieldFromValue :: Expr a+  }+  | Ternary+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeTernaryValue :: Expr a+  -- 'if' spaces+  , _unsafeTernaryWhitespaceIf :: [Whitespace]+  -- expr+  , _unsafeTernaryCond :: Expr a+  -- 'else' spaces+  , _unsafeTernaryWhitespaceElse :: [Whitespace]+  -- expr+  , _unsafeTernaryElse :: Expr a+  }+  | ListComp+  { _unsafeExprAnn :: a+  -- [ spaces+  , _unsafeListCompWhitespaceLeft :: [Whitespace]+  -- comprehension+  , _unsafeListCompValue :: Comprehension Expr a+  -- ] spaces+  , _unsafeListCompWhitespaceRight :: [Whitespace]+  }+  | List+  { _unsafeExprAnn :: a+  -- [ spaces+  , _unsafeListWhitespaceLeft :: [Whitespace]+  -- exprs+  , _unsafeListValues :: Maybe (CommaSep1' (Expr a))+  -- ] spaces+  , _unsafeListWhitespaceRight :: [Whitespace]+  }+  | DictComp+  { _unsafeExprAnn :: a+  -- { spaces+  , _unsafeDictCompWhitespaceLeft :: [Whitespace]+  -- comprehension+  , _unsafeDictCompValue :: Comprehension DictItem a+  -- } spaces+  , _unsafeDictCompWhitespaceRight :: [Whitespace]+  }+  | Dict+  { _unsafeExprAnn :: a+  , _unsafeDictWhitespaceLeft :: [Whitespace]+  , _unsafeDictValues :: Maybe (CommaSep1' (DictItem a))+  , _unsafeDictWhitespaceRight :: [Whitespace]+  }+  | SetComp+  { _unsafeExprAnn :: a+  -- { spaces+  , _unsafeSetCompWhitespaceLeft :: [Whitespace]+  -- comprehension+  , _unsafeSetCompValue :: Comprehension Expr a+  -- } spaces+  , _unsafeSetCompWhitespaceRight :: [Whitespace]+  }+  | Set+  { _unsafeExprAnn :: a+  , _unsafeSetWhitespaceLeft :: [Whitespace]+  , _unsafeSetValues :: CommaSep1' (Expr a)+  , _unsafeSetWhitespaceRight :: [Whitespace]+  }+  | Deref+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeDerefValueLeft :: Expr a+  -- . spaces+  , _unsafeDerefWhitespaceLeft :: [Whitespace]+  -- ident+  , _unsafeDerefValueRight :: Ident '[] a+  }+  | Subscript+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeSubscriptValueLeft :: Expr a+  -- [ spaces+  , _unsafeSubscriptWhitespaceLeft :: [Whitespace]+  -- expr+  , _unsafeSubscriptValueRight :: CommaSep1' (Subscript a)+  -- ] spaces+  , _unsafeSubscriptWhitespaceRight :: [Whitespace]+  }+  | Call+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeCallFunction :: Expr a+  -- ( spaces+  , _unsafeCallWhitespaceLeft :: [Whitespace]+  -- exprs+  , _unsafeCallArguments :: Maybe (CommaSep1' (Arg a))+  -- ) spaces+  , _unsafeCallWhitespaceRight :: [Whitespace]+  }+  | None+  { _unsafeExprAnn :: a+  , _unsafeNoneWhitespace :: [Whitespace]+  }+  | Ellipsis+  { _unsafeExprAnn :: a+  , _unsafeEllipsisWhitespace :: [Whitespace]+  }+  | BinOp+  { _unsafeExprAnn :: a+  , _unsafeBinOpExprLeft :: Expr a+  , _unsafeBinOpOp :: BinOp a+  , _unsafeBinOpExprRight :: Expr a+  }+  | UnOp+  { _unsafeExprAnn :: a+  , _unsafeUnOpOp :: UnOp a+  , _unsafeUnOpValue :: Expr a+  }+  | Parens+  { _unsafeExprAnn :: a+  -- ( spaces+  , _unsafeParensWhitespaceLeft :: [Whitespace]+  -- expr+  , _unsafeParensValue :: Expr a+  -- ) spaces+  , _unsafeParensWhitespaceAfter :: [Whitespace]+  }+  | Ident+  { _unsafeIdentValue :: Ident '[] a+  }+  | Int+  { _unsafeExprAnn :: a+  , _unsafeIntValue :: IntLiteral a+  , _unsafeIntWhitespace :: [Whitespace]+  }+  | Float+  { _unsafeExprAnn :: a+  , _unsafeFloatValue :: FloatLiteral a+  , _unsafeFloatWhitespace :: [Whitespace]+  }+  | Imag+  { _unsafeExprAnn :: a+  , _unsafeImagValue :: ImagLiteral a+  , _unsafeImagWhitespace :: [Whitespace]+  }+  | Bool+  { _unsafeExprAnn :: a+  , _unsafeBoolValue :: Bool+  , _unsafeBoolWhitespace :: [Whitespace]+  }+  | String+  { _unsafeExprAnn :: a+  , _unsafeStringLiteralValue :: NonEmpty (StringLiteral a)+  }+  | Tuple+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeTupleHead :: Expr a+  -- , spaces+  , _unsafeTupleWhitespace :: Comma+  -- [exprs]+  , _unsafeTupleTail :: Maybe (CommaSep1' (Expr a))+  }+  | Not+  { _unsafeExprAnn :: a+  , _unsafeNotWhitespace :: [Whitespace]+  , _unsafeNotValue :: Expr a+  }+  | Generator+  { _unsafeExprAnn :: a+  , _generatorValue :: Comprehension Expr a+  }+  | Await+  { _unsafeExprAnn :: a+  , _unsafeAwaitWhitespace :: [Whitespace]+  , _unsafeAwaitValue :: Expr a+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++exprAnn :: Lens' (Expr a) a+exprAnn =+  lens+    (\case+        Unit a _ _ -> a+        StarExpr a _ _ -> a+        Lambda a _ _ _ _ -> a+        Yield a _ _ -> a+        YieldFrom a _ _ _ -> a+        Ternary a _ _ _ _ _ -> a+        None a _ -> a+        Ellipsis a _ -> a+        List a _ _ _ -> a+        ListComp a _ _ _ -> a+        Deref a _ _ _ -> a+        Subscript a _ _ _ _ -> a+        Call a _ _ _ _ -> a+        BinOp a _ _ _ -> a+        UnOp a _ _ -> a+        Parens a _ _ _ -> a+        Ident a -> a ^. identAnn+        Int a _ _ -> a+        Float a _ _ -> a+        Imag a _ _ -> a+        Bool a _ _ -> a+        String a _ -> a+        Not a _ _ -> a+        Tuple a _ _ _ -> a+        DictComp a _ _ _ -> a+        Dict a _ _ _ -> a+        SetComp a _ _ _ -> a+        Set a _ _ _ -> a+        Generator a _ -> a+        Await a _ _ -> a)+    (\e ann ->+      case e of+        Unit _ a b -> Unit ann a b+        StarExpr _ a b -> StarExpr ann a b+        Lambda _ a b c d -> Lambda ann a b c d+        Yield _ a b -> Yield ann a b+        YieldFrom ann a b c -> YieldFrom ann a b c+        Ternary ann a b c d e -> Ternary ann a b c d e+        None _ a -> None ann a+        Ellipsis _ a -> Ellipsis ann a+        List _ a b c -> List ann a b c+        ListComp _ a b c -> ListComp ann a b c+        Deref _ a b c -> Deref ann a b c+        Subscript _ a b c d -> Subscript ann a b c d+        Call _ a b c d -> Call ann a b c d+        BinOp _ a b c -> BinOp ann a b c+        UnOp _ a b -> UnOp ann a b+        Parens _ a b c -> Parens ann a b c+        Ident a -> Ident $ a & identAnn .~ ann+        Int _ a b -> Int ann a b+        Float _ a b -> Float ann a b+        Imag _ a b -> Imag ann a b+        Bool _ a b -> Bool ann a b+        String _ a -> String ann a+        Not _ a b -> Not ann a b+        Tuple _ a b c -> Tuple ann a b c+        DictComp _ a b c -> DictComp ann a b c+        Dict _ a b c -> Dict ann a b c+        SetComp _ a b c -> SetComp ann a b c+        Set _ a b c -> Set ann a b c+        Generator _ a -> Generator ann a+        Await _ a b -> Not ann a b)++data Suite a+  -- ':' <space> smallStatement+  = SuiteOne a Colon (SmallStatement a)+  | SuiteMany a+      -- ':' <spaces> [comment] <newline>+      Colon (Maybe (Comment a)) Newline+      -- <block>+      (Block a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++data Block a+  = Block+  { _blockBlankLines :: [(Blank a, Newline)]+  , _blockHead :: Statement a+  , _blockTail :: [Either (Blank a, Newline) (Statement a)]+  } deriving (Eq, Show)++instance Functor Block where+  fmap f (Block a b c) =+    Block+      (over (mapped._1.mapped) f a)+      (fmap f b)+      (bimap (over (_1.mapped) f) (fmap f) <$> c)++instance Foldable Block where+  foldMap f (Block a b c) =+    foldMapOf (folded._1.folded) f a <>+    foldMap f b <>+    foldMap (bifoldMap (foldMapOf (_1.folded) f) (foldMap f)) c++instance Traversable Block where+  traverse f (Block a b c) =+    Block <$>+    traverseOf (traverse._1.traverse) f a <*>+    traverse f b <*>+    traverse (bitraverse (traverseOf (_1.traverse) f) (traverse f)) c++data WithItem a+  = WithItem+  { _withItemAnn :: a+  , _withItemValue :: Expr a+  , _withItemBinder :: Maybe ([Whitespace], Expr a)+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++data Decorator a+  = Decorator+  { _decoratorAnn :: a+  , _decoratorIndents :: Indents a+  , _decoratorAt :: At+  , _decoratorExpr :: Expr a+  , _decoratorComment :: Maybe (Comment a)+  , _decoratorNewline :: Newline+  , _decoratorBlankLines :: [(Blank a, Newline)]+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++data ExceptAs a+  = ExceptAs+  { _exceptAsAnn :: a+  , _exceptAsExpr :: Expr a+  , _exceptAsName :: Maybe ([Whitespace], Ident '[] a)+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++data Module a+  = ModuleEmpty+  | ModuleBlankFinal (Blank a)+  | ModuleBlank (Blank a) Newline (Module a)+  | ModuleStatement (Statement a) (Module a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++data FromIRContext+  = FromIRContext+  { _allowStarred :: Bool+  }++makeLenses ''FromIRContext++fromIR_expr+  :: AsIRError e a+  => Expr a+  -> Validation (NonEmpty e) (Syntax.Expr '[] a)+fromIR_expr ex =+  case ex of+    StarExpr{} -> Failure $ pure (_InvalidUnpacking # (ex ^. exprAnn))+    Unit a b c -> pure $ Syntax.Unit a b c+    Lambda a b c d e ->+      (\c' -> Syntax.Lambda a b c' d) <$>+      traverse fromIR_param c <*>+      fromIR_expr e+    Yield a b c -> Syntax.Yield a b <$> traverse fromIR_expr c+    YieldFrom a b c d -> Syntax.YieldFrom a b c <$> fromIR_expr d+    Ternary a b c d e f ->+      (\b' d' -> Syntax.Ternary a b' c d' e) <$>+      fromIR_expr b <*>+      fromIR_expr d <*>+      fromIR_expr f+    ListComp a b c d ->+      (\c' -> Syntax.ListComp a b c' d) <$>+      fromIR_comprehension fromIR_expr c+    List a b c d ->+      (\c' -> Syntax.List a b c' d) <$>+      traverseOf (traverse.traverse) fromIR_listItem c+    DictComp a b c d ->+      (\c' -> Syntax.DictComp a b c' d) <$>+      fromIR_comprehension fromIR_dictItem c+    Dict a b c d ->+      (\c' -> Syntax.Dict a b c' d) <$>+      traverseOf (traverse.traverse) fromIR_dictItem c+    SetComp a b c d ->+      (\c' -> Syntax.SetComp a b c' d) <$>+      fromIR_comprehension fromIR_setItem c+    Set a b c d ->+      (\c' -> Syntax.Set a b c' d) <$>+      traverse fromIR_setItem c+    Deref a b c d ->+      (\b' -> Syntax.Deref a b' c d) <$>+      fromIR_expr b+    Subscript a b c d e ->+      (\b' d' -> Syntax.Subscript a b' c d' e) <$>+      fromIR_expr b <*>+      traverse fromIR_subscript d+    Call a b c d e ->+      (\b' d' -> Syntax.Call a b' c d' e) <$>+      fromIR_expr b <*>+      traverseOf (traverse.traverse) fromIR_arg d+    None a b -> pure $ Syntax.None a b+    Ellipsis a b -> pure $ Syntax.Ellipsis a b+    BinOp a b c d ->+      (\b' d' -> Syntax.BinOp a b' c d') <$>+      fromIR_expr b <*>+      fromIR_expr d+    UnOp a b c ->+      Syntax.UnOp a b <$> fromIR_expr c+    Parens a b c d ->+      (\c' -> Syntax.Parens a b c' d) <$>+      fromIR_expr c+    Ident a -> pure $ Syntax.Ident a+    Int a b c -> pure $ Syntax.Int a b c+    Float a b c -> pure $ Syntax.Float a b c+    Imag a b c -> pure $ Syntax.Imag a b c+    Bool a b c -> pure $ Syntax.Bool a b c+    String a b -> pure $ Syntax.String a b+    Tuple a b c d ->+      (\b' -> Syntax.Tuple a b' c) <$>+      fromIR_tupleItem b <*>+      traverseOf (traverse.traverse) fromIR_tupleItem d+    Not a b c -> Syntax.Not a b <$> fromIR_expr c+    Generator a b -> Syntax.Generator a <$> fromIR_comprehension fromIR_expr b+    Await a b c -> Syntax.Await a b <$> fromIR_expr c++fromIR_suite+  :: AsIRError e a+  => Suite a+  -> Validation (NonEmpty e) (Syntax.Suite '[] a)+fromIR_suite s =+  case s of+    SuiteOne a b c ->+      Syntax.SuiteOne a b <$> fromIR_smallStatement c+    SuiteMany a b c d e ->+      Syntax.SuiteMany a b c d <$> fromIR_block e++fromIR_param+  :: AsIRError e a+  => Param a+  -> Validation (NonEmpty e) (Syntax.Param '[] a)+fromIR_param p =+  case p of+    PositionalParam a b c ->+      Syntax.PositionalParam a b <$> traverseOf (traverse._2) fromIR_expr c+    KeywordParam a b c d e ->+      Syntax.KeywordParam a b <$>+      traverseOf (traverse._2) fromIR_expr c <*>+      pure d <*>+      fromIR_expr e+    StarParam a b c d ->+      Syntax.StarParam a b c <$> traverseOf (traverse._2) fromIR_expr d+    UnnamedStarParam a b -> pure $ Syntax.UnnamedStarParam a b+    DoubleStarParam a b c d ->+      Syntax.DoubleStarParam a b c <$> traverseOf (traverse._2) fromIR_expr d++fromIR_arg+  :: AsIRError e a+  => Arg a+  -> Validation (NonEmpty e) (Syntax.Arg '[] a)+fromIR_arg a =+  case a of+    PositionalArg a b -> Syntax.PositionalArg a <$> fromIR_expr b+    KeywordArg a b c d -> Syntax.KeywordArg a b c <$> fromIR_expr d+    StarArg a b c -> Syntax.StarArg a b <$> fromIR_expr c+    DoubleStarArg a b c -> Syntax.DoubleStarArg a b <$> fromIR_expr c++fromIR_decorator+  :: AsIRError e a+  => Decorator a+  -> Validation (NonEmpty e) (Syntax.Decorator '[] a)+fromIR_decorator (Decorator a b c d e f g) =+  (\d' -> Syntax.Decorator a b c d' e f g) <$>+  fromIR_expr d++fromIR_exceptAs+  :: AsIRError e a+  => ExceptAs a+  -> Validation (NonEmpty e) (Syntax.ExceptAs '[] a)+fromIR_exceptAs (ExceptAs a b c) =+  (\b' -> Syntax.ExceptAs a b' c) <$>+  fromIR_expr b++fromIR_withItem+  :: AsIRError e a+  => WithItem a+  -> Validation (NonEmpty e) (Syntax.WithItem '[] a)+fromIR_withItem (WithItem a b c) =+  Syntax.WithItem a <$>+  fromIR_expr b <*>+  traverseOf (traverse._2) fromIR_expr c++fromIR_comprehension+  :: AsIRError e a+  => (ex a -> Validation (NonEmpty e) (ex' '[] a))+  -> Comprehension ex a+  -> Validation (NonEmpty e) (Syntax.Comprehension ex' '[] a)+fromIR_comprehension f (Comprehension a b c d) =+  Syntax.Comprehension a <$>+  f b <*>+  fromIR_compFor c <*>+  traverse (bitraverse fromIR_compFor fromIR_compIf) d++fromIR_dictItem+  :: AsIRError e a+  => DictItem a+  -> Validation (NonEmpty e) (Syntax.DictItem '[] a)+fromIR_dictItem di =+  case di of+    DictItem a b c d ->+      (\b' -> Syntax.DictItem a b' c) <$>+      fromIR_expr b <*>+      fromIR_expr d+    DictUnpack a b c ->+      Syntax.DictUnpack a b <$> fromIR_expr c++fromIR_subscript+  :: AsIRError e a+  => Subscript a+  -> Validation (NonEmpty e) (Syntax.Subscript '[] a)+fromIR_subscript s =+  case s of+    SubscriptExpr a -> Syntax.SubscriptExpr <$> fromIR_expr a+    SubscriptSlice a b c d ->+      (\a' -> Syntax.SubscriptSlice a' b) <$>+      traverse fromIR_expr a <*>+      traverse fromIR_expr c <*>+      traverseOf (traverse._2.traverse) fromIR_expr d++fromIR_block+  :: AsIRError e a+  => Block a+  -> Validation (NonEmpty e) (Syntax.Block '[] a)+fromIR_block (Block a b c) =+  Syntax.Block a <$>+  fromIR_statement b <*>+  traverseOf (traverse.traverse) fromIR_statement c++fromIR_compFor+  :: AsIRError e a+  => CompFor a+  -> Validation (NonEmpty e) (Syntax.CompFor '[] a)+fromIR_compFor (CompFor a b c d e) =+  (\c' -> Syntax.CompFor a b c' d) <$>+  fromIR_expr c <*>+  fromIR_expr e++fromIR_compIf+  :: AsIRError e a+  => CompIf a+  -> Validation (NonEmpty e) (Syntax.CompIf '[] a)+fromIR_compIf (CompIf a b c) =+  Syntax.CompIf a b <$> fromIR_expr c++fromIR_smallStatement+  :: AsIRError e a+  => SmallStatement a+  -> Validation (NonEmpty e) (Syntax.SmallStatement '[] a)+fromIR_smallStatement (MkSmallStatement b c d e f) =+  (\b' c' -> Syntax.MkSmallStatement b' c' d e f) <$>+  fromIR_SimpleStatement b <*>+  traverseOf (traverse._2) fromIR_SimpleStatement c++fromIR_statement+  :: AsIRError e a+  => Statement a+  -> Validation (NonEmpty e) (Syntax.Statement '[] a)+fromIR_statement ex =+  case ex of+    SmallStatement i a ->+      Syntax.SmallStatement i <$> fromIR_smallStatement a+    CompoundStatement a ->+      Syntax.CompoundStatement <$> fromIR_compoundStatement a++fromIR_SimpleStatement+  :: AsIRError e a+  => SimpleStatement a+  -> Validation (NonEmpty e) (Syntax.SimpleStatement '[] a)+fromIR_SimpleStatement ex =+  case ex of+    Assign a b c ->+      Syntax.Assign a <$>+      fromIR_expr b <*>+      traverseOf (traverse._2) fromIR_expr c+    Return a b c -> Syntax.Return a b <$> traverse fromIR_expr c+    Expr a b -> Syntax.Expr a <$> fromIR_expr b+    AugAssign a b c d ->+      (\b' d' -> Syntax.AugAssign a b' c d') <$>+      fromIR_expr b <*>+      fromIR_expr d+    Pass a ws -> pure $ Syntax.Pass a ws+    Break a ws -> pure $ Syntax.Break a ws+    Continue a ws -> pure $ Syntax.Continue a ws+    Global a b c -> pure $ Syntax.Global a b c+    Nonlocal a b c -> pure $ Syntax.Nonlocal a b c+    Del a b c -> Syntax.Del a b <$> traverse fromIR_expr c+    Import a b c -> pure $ Syntax.Import a b c+    From a b c d e -> pure $ Syntax.From a b c d e+    Raise a b c ->+      Syntax.Raise a b <$>+      traverse+        (\(a, b) -> (,) <$>+          fromIR_expr a <*>+          traverseOf (traverse._2) fromIR_expr b)+        c+    Assert a b c d ->+      Syntax.Assert a b <$>+      fromIR_expr c <*>+      traverseOf (traverse._2) fromIR_expr d++fromIR_compoundStatement+  :: AsIRError e a+  => CompoundStatement a+  -> Validation (NonEmpty e) (Syntax.CompoundStatement '[] a)+fromIR_compoundStatement st =+  case st of+    Fundef a b asyncWs c d e f g h i j ->+      (\b' g' i' -> Syntax.Fundef a b' asyncWs c d e f g' h i') <$>+      traverse fromIR_decorator b <*>+      traverse fromIR_param g <*>+      traverseOf (traverse._2) fromIR_expr i <*>+      fromIR_suite j+    If a b c d e f g ->+      Syntax.If a b c <$>+      fromIR_expr d <*>+      fromIR_suite e <*>+      traverse (\(a, b, c, d) -> (,,,) a b <$> fromIR_expr c <*> fromIR_suite d) f <*>+      traverseOf (traverse._3) fromIR_suite g+    While a b c d e f ->+      Syntax.While a b c <$>+      fromIR_expr d <*>+      fromIR_suite e <*>+      traverseOf (traverse._3) fromIR_suite f+    TryExcept a b c d e f g ->+      Syntax.TryExcept a b c <$>+      fromIR_suite d <*>+      traverse+        (\(a, b, c, d) -> (,,,) a b <$> traverse fromIR_exceptAs c <*> fromIR_suite d)+        e <*>+      traverseOf (traverse._3) fromIR_suite f <*>+      traverseOf (traverse._3) fromIR_suite g+    TryFinally a b c d e f g ->+      (\d' -> Syntax.TryFinally a b c d' e f) <$> fromIR_suite d <*> fromIR_suite g+    For a b asyncWs c d e f g h ->+      (\d' -> Syntax.For a b asyncWs c d' e) <$>+      fromIR_expr d <*>+      traverse fromIR_expr f <*>+      fromIR_suite g <*>+      traverseOf (traverse._3) fromIR_suite h+    ClassDef a b c d e f g ->+      (\b' -> Syntax.ClassDef a b' c d e) <$>+      traverse fromIR_decorator b <*>+      traverseOf (traverse._2.traverse.traverse) fromIR_arg f <*>+      fromIR_suite g+    With a b asyncWs c d e ->+      Syntax.With a b asyncWs c <$>+      traverse fromIR_withItem d <*>+      fromIR_suite e++fromIR_listItem+  :: AsIRError e a+  => Expr a+  -> Validation (NonEmpty e) (Syntax.ListItem '[] a)+fromIR_listItem (StarExpr a b c) =+  Syntax.ListUnpack a [] b <$> fromIR_expr c+fromIR_listItem (Parens a b c d) =+  (\case+      Syntax.ListUnpack w x y z -> Syntax.ListUnpack w ((b, d) : x) y z+      Syntax.ListItem x y -> Syntax.ListItem a (Syntax.Parens x b y d)) <$>+  fromIR_listItem c+fromIR_listItem e = (\x -> Syntax.ListItem (x ^. Syntax.exprAnn) x) <$> fromIR_expr e++fromIR_tupleItem+  :: AsIRError e a+  => Expr a+  -> Validation (NonEmpty e) (Syntax.TupleItem '[] a)+fromIR_tupleItem (StarExpr a b c) =+  Syntax.TupleUnpack a [] b <$> fromIR_expr c+fromIR_tupleItem (Parens a b c d) =+  (\case+      Syntax.TupleUnpack w x y z -> Syntax.TupleUnpack w ((b, d) : x) y z+      Syntax.TupleItem x y -> Syntax.TupleItem a (Syntax.Parens x b y d)) <$>+  fromIR_tupleItem c+fromIR_tupleItem e =+  (\x -> Syntax.TupleItem (x ^. Syntax.exprAnn) x) <$> fromIR_expr e++fromIR_setItem+  :: AsIRError e a+  => Expr a+  -> Validation (NonEmpty e) (Syntax.SetItem '[] a)+fromIR_setItem (StarExpr a b c) =+  Syntax.SetUnpack a [] b <$> fromIR_expr c+fromIR_setItem (Parens a b c d) =+  (\case+      Syntax.SetUnpack w x y z -> Syntax.SetUnpack w ((b, d) : x) y z+      Syntax.SetItem x y -> Syntax.SetItem a (Syntax.Parens x b y d)) <$>+  fromIR_setItem c+fromIR_setItem e = (\x -> Syntax.SetItem (x ^. Syntax.exprAnn) x) <$> fromIR_expr e++fromIR+  :: AsIRError e a+  => Module a+  -> Validation (NonEmpty e) (Syntax.Module '[] a)+fromIR ModuleEmpty = pure Syntax.ModuleEmpty+fromIR (ModuleBlankFinal a) = pure $ Syntax.ModuleBlankFinal a+fromIR (ModuleBlank a b c) = Syntax.ModuleBlank a b <$> fromIR c+fromIR (ModuleStatement a b) = Syntax.ModuleStatement <$> fromIR_statement a <*> fromIR b
+ src/Language/Python/Internal/Token.hs view
@@ -0,0 +1,230 @@+{-# language DeriveFunctor #-}+{-# language OverloadedStrings #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Internal.Token+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Internal.Token where++import Data.Deriving (deriveEq1, deriveOrd1)+import Data.Functor.Classes (liftCompare, liftEq)++import Language.Python.Syntax.Comment (Comment(..))+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace (Newline(..), Indents)++-- | A 'PyToken' is a single lexical token of Python source. A 'PyToken' has an+-- optional annotation, which can be '()' when no annotation is desired.+data PyToken a+  = TkIf a+  | TkElse a+  | TkElif a+  | TkWhile a+  | TkAssert a+  | TkDef a+  | TkReturn a+  | TkPass a+  | TkBreak a+  | TkContinue a+  | TkTrue a+  | TkFalse a+  | TkNone a+  | TkEllipsis a+  | TkOr a+  | TkAnd a+  | TkIs a+  | TkNot a+  | TkGlobal a+  | TkNonlocal a+  | TkDel a+  | TkLambda a+  | TkImport a+  | TkFrom a+  | TkAs a+  | TkRaise a+  | TkTry a+  | TkExcept a+  | TkFinally a+  | TkClass a+  | TkRightArrow a+  | TkWith a+  | TkFor a+  | TkIn a+  | TkYield a+  | TkInt (IntLiteral a)+  | TkFloat (FloatLiteral a)+  | TkImag (ImagLiteral a)+  | TkIdent String a+  | TkString (Maybe StringPrefix) StringType QuoteType [PyChar] a+  | TkBytes BytesPrefix StringType QuoteType [PyChar] a+  | TkRawString RawStringPrefix StringType QuoteType [PyChar] a+  | TkRawBytes RawBytesPrefix StringType QuoteType [PyChar] a+  | TkSpace a+  | TkTab a+  | TkNewline Newline a+  | TkLeftBracket a+  | TkRightBracket a+  | TkLeftParen a+  | TkRightParen a+  | TkLeftBrace a+  | TkRightBrace a+  | TkLt a+  | TkLte a+  | TkEq a+  | TkDoubleEq a+  | TkBangEq a+  | TkGt a+  | TkGte a+  | TkContinued Newline a+  | TkColon a+  | TkSemicolon a+  | TkComma a+  | TkDot a+  | TkPlus a+  | TkMinus a+  | TkTilde a+  | TkComment (Comment a)+  | TkStar a+  | TkDoubleStar a+  | TkSlash a+  | TkDoubleSlash a+  | TkPercent a+  | TkShiftLeft a+  | TkShiftRight a+  | TkPlusEq a+  | TkMinusEq a+  | TkStarEq a+  | TkAtEq a+  | TkAt a+  | TkSlashEq a+  | TkPercentEq a+  | TkAmpersandEq a+  | TkPipeEq a+  | TkCaretEq a+  | TkShiftLeftEq a+  | TkShiftRightEq a+  | TkDoubleStarEq a+  | TkDoubleSlashEq a+  | TkPipe a+  | TkCaret a+  | TkAmpersand a+  | TkIndent a (Indents a)+  | TkLevel a (Indents a)+  | TkDedent a+  deriving (Show, Functor)+deriveEq1 ''PyToken+deriveOrd1 ''PyToken++instance Eq (PyToken a) where+  (==) = liftEq (\_ _ -> True)++instance Ord (PyToken a) where+  compare = liftCompare (\_ _ -> EQ)++-- | Get the annotation from a 'PyToken'.+pyTokenAnn :: PyToken a -> a+pyTokenAnn tk =+  case tk of+    TkPipe a -> a+    TkCaret a -> a+    TkAmpersand a -> a+    TkIndent a _ -> a+    TkLevel a _ -> a+    TkDedent a -> a+    TkDef a -> a+    TkReturn a -> a+    TkPass a -> a+    TkBreak a -> a+    TkContinue a -> a+    TkTrue a -> a+    TkFalse a -> a+    TkNone a -> a+    TkEllipsis a -> a+    TkOr a -> a+    TkAnd a -> a+    TkIs a -> a+    TkNot a -> a+    TkGlobal a -> a+    TkNonlocal a -> a+    TkDel a -> a+    TkLambda a -> a+    TkImport a -> a+    TkFrom a -> a+    TkAs a -> a+    TkRaise a -> a+    TkTry a -> a+    TkExcept a -> a+    TkFinally a -> a+    TkClass a -> a+    TkRightArrow a -> a+    TkWith a -> a+    TkFor a -> a+    TkIn a -> a+    TkYield a -> a+    TkPlus a -> a+    TkMinus a -> a+    TkTilde a -> a+    TkIf a -> a+    TkElse a -> a+    TkElif a -> a+    TkWhile a -> a+    TkAssert a -> a+    TkInt a -> _intLiteralAnn a+    TkFloat a -> _floatLiteralAnn a+    TkImag a -> _imagLiteralAnn a+    TkIdent _ a -> a+    TkString _ _ _ _ a -> a+    TkBytes _ _ _ _ a -> a+    TkRawString _ _ _ _ a -> a+    TkRawBytes _ _ _ _ a -> a+    TkSpace a -> a+    TkTab a -> a+    TkNewline _ a -> a+    TkLeftBracket a -> a+    TkRightBracket a -> a+    TkLeftParen a -> a+    TkRightParen a -> a+    TkLeftBrace a -> a+    TkRightBrace a -> a+    TkLt a -> a+    TkLte a -> a+    TkEq a -> a+    TkDoubleEq a -> a+    TkBangEq a -> a+    TkGt a -> a+    TkGte a -> a+    TkContinued _ a -> a+    TkColon a -> a+    TkSemicolon a -> a+    TkComma a -> a+    TkDot a -> a+    TkComment a -> _commentAnn a+    TkStar a -> a+    TkDoubleStar a -> a+    TkSlash a -> a+    TkDoubleSlash a -> a+    TkPercent a -> a+    TkShiftLeft a -> a+    TkShiftRight a -> a+    TkPlusEq a -> a+    TkMinusEq a -> a+    TkStarEq a -> a+    TkAtEq a -> a+    TkAt a -> a+    TkSlashEq a -> a+    TkPercentEq a -> a+    TkAmpersandEq a -> a+    TkPipeEq a -> a+    TkCaretEq a -> a+    TkShiftLeftEq a -> a+    TkShiftRightEq a -> a+    TkDoubleStarEq a -> a+    TkDoubleSlashEq a -> a
+ src/Language/Python/Optics.hs view
@@ -0,0 +1,453 @@+{-# language DataKinds #-}+{-# language FlexibleContexts #-}+{-# language MultiParamTypeClasses #-}+{-# language FlexibleInstances #-}+{-# language InstanceSigs, ScopedTypeVariables #-}+{-# language PolyKinds #-}+{-# language LambdaCase #-}++{-|+Module      : Language.Python.Optics+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Optics for manipulating Python syntax trees++-}++module Language.Python.Optics+  ( module Language.Python.Optics.Validated+    -- * Indentation+  , module Language.Python.Optics.Indents+    -- * Newlines+  , module Language.Python.Optics.Newlines+    -- * Simple statements+    -- ** Assignment+  , assignTargets+    -- * Compound statements+  , HasCompoundStatement(..)+    -- ** Function definitions+  , HasFundef(..)+    -- ** Class defintions+  , HasClassDef(..)+    -- ** @while@ statements+  , HasWhile(..)+    -- ** @for@ statements+  , HasFor(..)+    -- ** @with@ statements+  , HasWith(..)+    -- ** @if@ statements+  , HasIf(..)+  , _Elif+    -- ** @try@ statements+  , HasTryExcept(..)+  , HasTryFinally(..)+  , _Finally+  , _Except+    -- ** @else@+  , _Else+    -- * Parameters+  , _PositionalParam+  , _KeywordParam+  , _UnnamedStarParam+  , _StarParam+    -- * Expressions+    -- ** Identifiers+  , _Ident+    -- ** @None@+  , _None+    -- ** Function calls+  , _Call+    -- ** Tuples+  , _Tuple+  , _TupleUnpack+  , tupleItems+    -- ** Lists+  , _List+  , _ListUnpack+  , listItems+  )+where++import Control.Lens.Getter ((^.), view)+import Control.Lens.Iso (Iso', iso, from)+import Control.Lens.Traversal (Traversal)+import Control.Lens.Prism (Prism, prism)++import Language.Python.Optics.Indents+import Language.Python.Optics.Newlines+import Language.Python.Optics.Validated+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Types+import Language.Python.Syntax.Whitespace++_TupleUnpack :: Prism (TupleItem v a) (TupleItem '[] a) (TupleUnpack v a) (TupleUnpack '[] a)+_TupleUnpack =+  prism+    (\(MkTupleUnpack a b c d) -> TupleUnpack a b c d)+    (\case+       TupleUnpack a b c d -> Right $ MkTupleUnpack a b c d+       a -> Left $ a ^. unvalidated)++_Tuple :: Prism (Expr v a) (Expr '[] a) (Tuple v a) (Tuple '[] a)+_Tuple =+  prism+    (\(MkTuple a b c d) -> Tuple a b c d)+    (\case+        Tuple a b c d -> Right (MkTuple a b c d)+        a -> Left $ a ^. unvalidated)++tupleItems :: Traversal (Tuple v a) (Tuple '[] a) (TupleItem v a) (TupleItem '[] a)+tupleItems f (MkTuple a b c d) =+  (\b' d' -> MkTuple a b' c d') <$>+  f b <*>+  (traverse.traverse) f d++_ListUnpack :: Prism (ListItem v a) (ListItem '[] a) (ListUnpack v a) (ListUnpack '[] a)+_ListUnpack =+  prism+    (\(MkListUnpack a b c d) -> ListUnpack a b c d)+    (\case+       ListUnpack a b c d -> Right $ MkListUnpack a b c d+       a -> Left $ a ^. unvalidated)++_List :: Prism (Expr v a) (Expr '[] a) (List v a) (List '[] a)+_List =+  prism+    (\(MkList a b c d) -> List a b c d)+    (\case+        List a b c d -> Right (MkList a b c d)+        a -> Left $ a ^. unvalidated)++listItems :: Traversal (List v a) (List '[] a) (ListItem v a) (ListItem '[] a)+listItems f (MkList a b c d) =+  (\c' -> MkList a b c' d) <$>+  (traverse.traverse) f c++_None :: Prism (Expr v a) (Expr '[] a) (None v a) (None '[] a)+_None =+  prism+    (\(MkNone a b) -> None a b)+    (\case+        None a b -> Right (MkNone a b)+        a -> Left $ a ^. unvalidated)++_KeywordParam+  :: Prism+       (Param v a)+       (Param '[] a)+       (KeywordParam v a)+       (KeywordParam '[] a)+_KeywordParam =+  prism+    (\(MkKeywordParam a b c d e) -> KeywordParam a b c d e)+    (\case+        KeywordParam a b c d e -> Right (MkKeywordParam a b c d e)+        a -> Left $ a ^. unvalidated)++_PositionalParam+  :: Prism+       (Param v a)+       (Param '[] a)+       (PositionalParam v a)+       (PositionalParam '[] a)+_PositionalParam =+  prism+    (\(MkPositionalParam a b c) -> PositionalParam a b c)+    (\case+        PositionalParam a b c -> Right (MkPositionalParam a b c)+        a -> Left $ a ^. unvalidated)++_StarParam+  :: Prism+       (Param v a)+       (Param '[] a)+       (StarParam v a)+       (StarParam '[] a)+_StarParam =+  prism+    (\(MkStarParam a b c d) -> StarParam a b c d)+    (\case+        StarParam a b c d -> Right (MkStarParam a b c d)+        a -> Left $ a ^. unvalidated)++_UnnamedStarParam+  :: Prism+       (Param v a)+       (Param '[] a)+       (UnnamedStarParam v a)+       (UnnamedStarParam '[] a)+_UnnamedStarParam =+  prism+    (\(MkUnnamedStarParam a b) -> UnnamedStarParam a b)+    (\case+        UnnamedStarParam a b -> Right (MkUnnamedStarParam a b)+        a -> Left $ a ^. unvalidated)++class HasCompoundStatement s where+  _CompoundStatement :: Prism (s v a) (s '[] a) (CompoundStatement v a) (CompoundStatement '[] a)++instance HasCompoundStatement CompoundStatement where+  _CompoundStatement = id++instance HasCompoundStatement Statement where+  _CompoundStatement =+    prism+      CompoundStatement+      (\case+          CompoundStatement a -> Right a+          a -> Left (a ^. unvalidated))++class HasFundef s where+  _Fundef :: Prism (s v a) (s '[] a) (Fundef v a) (Fundef '[] a)++instance HasFundef Fundef where+  _Fundef = id++instance HasFundef CompoundStatement where+  _Fundef =+    prism+      (\(MkFundef idnt a b c d e f g h i j) ->+         Fundef idnt a b c d e f g h i j)+      (\case+          Fundef idnt a b c d e f g h i j ->+            Right $ MkFundef idnt a b c d e f g h i j+          a -> Left $ a ^. unvalidated)++instance HasFundef Statement where+  _Fundef = _CompoundStatement._Fundef++class HasWhile s where+  _While :: Prism (s v a) (s '[] a) (While v a) (While '[] a)++instance HasWhile While where+  _While = id++instance HasWhile CompoundStatement where+  _While =+    prism+      (\(MkWhile a b c d e f) ->+        While a b c d e $ view _Else <$> f)+      (\case+          While a b c d e f ->+            Right . MkWhile a b c d e $ view (from _Else) <$> f+          a -> Left $ a ^. unvalidated)++instance HasWhile Statement where+  _While = _CompoundStatement._While++_Else :: Iso' (Else v a) (Indents a, [Whitespace], Suite v a)+_Else = iso (\(MkElse a b c) -> (a, b, c)) (\(a, b, c) -> MkElse a b c)++_Elif :: Iso' (Elif v a) (Indents a, [Whitespace], Expr v a, Suite v a)+_Elif = iso (\(MkElif a b c d) -> (a, b, c, d)) (\(a, b, c, d) -> MkElif a b c d)++_Finally :: Iso' (Finally v a) (Indents a, [Whitespace], Suite v a)+_Finally = iso (\(MkFinally a b c) -> (a, b, c)) (\(a, b, c) -> MkFinally a b c)++_Except :: Iso' (Except v a) (Indents a, [Whitespace], Maybe (ExceptAs v a), Suite v a)+_Except = iso (\(MkExcept a b c d) -> (a, b, c, d)) (\(a, b, c, d) -> MkExcept a b c d)++class HasIf s where+  _If :: Prism (s v a) (s '[] a) (If v a) (If '[] a)++instance HasIf If where+  _If = id++instance HasIf CompoundStatement where+  _If =+    prism+      (\(MkIf a b c d e f g) ->+        If a b c d e (view _Elif <$> f) (view _Else <$> g))+      (\case+          If a b c d e f g ->+            Right $ MkIf a b c d e (view (from _Elif) <$> f) (view (from _Else) <$> g)+          a -> Left $ a ^. unvalidated)++instance HasIf Statement where+  _If = _CompoundStatement._If++class HasTryExcept s where+  _TryExcept :: Prism (s v a) (s '[] a) (TryExcept v a) (TryExcept '[] a)++instance HasTryExcept TryExcept where+  _TryExcept = id++instance HasTryExcept CompoundStatement where+  _TryExcept =+    prism+      (\(MkTryExcept a b c d e f g) ->+        TryExcept a b c d (view _Except <$> e) (view _Else <$> f) (view _Finally <$> g))+      (\case+          TryExcept a b c d e f g ->+            Right $+            MkTryExcept a b c d+              (view (from _Except) <$> e)+              (view (from _Else) <$> f)+              (view (from _Finally) <$> g)+          a -> Left $ a ^. unvalidated)++instance HasTryExcept Statement where+  _TryExcept = _CompoundStatement._TryExcept++class HasTryFinally s where+  _TryFinally :: Prism (s v a) (s '[] a) (TryFinally v a) (TryFinally '[] a)++instance HasTryFinally TryFinally where+  _TryFinally = id++instance HasTryFinally CompoundStatement where+  _TryFinally =+    prism+      (\(MkTryFinally a b c d e) ->+        (\(x, y, z) -> TryFinally a b c d x y z) (e ^. _Finally))+      (\case+          TryFinally a b c d e f g ->+            Right $ MkTryFinally a b c d ((e, f, g) ^. from _Finally)+          a -> Left $ a ^. unvalidated)++instance HasTryFinally Statement where+  _TryFinally = _CompoundStatement._TryFinally++class HasFor s where+  _For :: Prism (s v a) (s '[] a) (For v a) (For '[] a)++instance HasFor For where+  _For = id++instance HasFor CompoundStatement where+  _For =+    prism+      (\(MkFor a b c d e f g h i) ->+        For a b c d e f g h (view _Else <$> i))+      (\case+          For a b c d e f g h i ->+            Right $ MkFor a b c d e f g h (view (from _Else) <$> i)+          a -> Left $ a ^. unvalidated)++instance HasFor Statement where+  _For = _CompoundStatement._For++_Call :: Prism (Expr v a) (Expr '[] a) (Call v a) (Call '[] a)+_Call =+  prism+    (\(MkCall a b c d e) -> Call a b c d e)+    (\case+        Call a b c d e -> Right $ MkCall a b c d e+        a -> Left $ a ^. unvalidated)++class HasClassDef s where+  _ClassDef :: Prism (s v a) (s '[] a) (ClassDef v a) (ClassDef '[] a)++instance HasClassDef ClassDef where+  _ClassDef = id++instance HasClassDef CompoundStatement where+  _ClassDef =+    prism+      (\(MkClassDef a b c d e f g) -> ClassDef a b c d e f g)+      (\case+          ClassDef a b c d e f g -> Right $ MkClassDef a b c d e f g+          a -> Left $ a ^. unvalidated)++instance HasClassDef Statement where+  _ClassDef = _CompoundStatement._ClassDef++class HasWith s where+  _With :: Prism (s v a) (s '[] a) (With v a) (With '[] a)++instance HasWith With where+  _With = id++instance HasWith CompoundStatement where+  _With =+    prism+      (\(MkWith a b c d e f) -> With a b c d e f)+      (\case+          With a b c d e f -> Right $ MkWith a b c d e f+          a -> Left $ a ^. unvalidated)++instance HasWith Statement where+  _With = _CompoundStatement._With++_Ident :: Prism (Expr v a) (Expr '[] a) (Ident v a) (Ident '[] a)+_Ident =+  prism+    Ident+    (\case+        Ident a -> Right a+        a -> Left $ a ^. unvalidated)++-- | 'Traversal' targeting the variables that would modified as a result of an assignment+--+-- Here are some examples of assignment targets:+--+-- @+-- a = b+-- ^+-- @+--+-- @+-- (a, b, c) = d+--  ^  ^  ^+-- @+--+-- @+-- [a, b, *c] = d+--  ^  ^   ^+-- @+--+-- These expressions have variables on the left hand side of the @=@, but those variables+-- don't count as assignment targets:+--+-- @+-- a[b] = c+-- @+--+-- @+-- a(b) = c+-- @+--+-- @+-- {a: b} = c+-- @+assignTargets :: Traversal (Expr v a) (Expr '[] a) (Ident v a) (Ident '[] a)+assignTargets f e =+  case e of+    List a b c d -> (\c' -> List a b c' d) <$> (traverse.traverse._Exprs.assignTargets) f c+    Parens a b c d -> (\c' -> Parens a b c' d) <$> assignTargets f c+    Ident a -> Ident <$> f a+    Tuple a b c d ->+      (\b' d' -> Tuple a b' c d') <$>+      (_Exprs.assignTargets) f b <*>+      (traverse.traverse._Exprs.assignTargets) f d+    Unit{} -> pure $ e ^. unvalidated+    Lambda{} -> pure $ e ^. unvalidated+    Yield{} -> pure $ e ^. unvalidated+    YieldFrom{} -> pure $ e ^. unvalidated+    Ternary{} -> pure $ e ^. unvalidated+    ListComp{} -> pure $ e ^. unvalidated+    Deref{} -> pure $ e ^. unvalidated+    Subscript{} -> pure $ e ^. unvalidated+    Call{} -> pure $ e ^. unvalidated+    None{} -> pure $ e ^. unvalidated+    Ellipsis{} -> pure $ e ^. unvalidated+    BinOp{} -> pure $ e ^. unvalidated+    UnOp{} -> pure $ e ^. unvalidated+    Int{} -> pure $ e ^. unvalidated+    Float{} -> pure $ e ^. unvalidated+    Imag{} -> pure $ e ^. unvalidated+    Bool{} -> pure $ e ^. unvalidated+    String{} -> pure $ e ^. unvalidated+    Not{} -> pure $ e ^. unvalidated+    DictComp{} -> pure $ e ^. unvalidated+    Dict{} -> pure $ e ^. unvalidated+    SetComp{} -> pure $ e ^. unvalidated+    Set{} -> pure $ e ^. unvalidated+    Generator{} -> pure $ e ^. unvalidated+    Await{} -> pure $ e ^. unvalidated
+ src/Language/Python/Optics/Indents.hs view
@@ -0,0 +1,262 @@+{-# language DataKinds #-}+{-# language FlexibleInstances, FunctionalDependencies, MultiParamTypeClasses #-}++{-|+Module      : Language.Python.Optics.Indents+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Optics.Indents where++import Control.Lens.Traversal (Traversal')++import Language.Python.Syntax+import Language.Python.Internal.Token++-- | 'Traversal'' targeting the indent-chunks in a structure+--+-- e.g.+--+-- This is one indent chunk:+--+-- @+-- def a():+--     pass+--     if b:+--         pass+-- ^^^^+-- @+--+-- and this is another+--+-- @+-- def a():+--     pass+--     if b:+--         pass+--     ^^^^+-- @+_Indent :: HasIndents s a => Traversal' s [Whitespace]+_Indent = _Indents.indentsValue.traverse.indentWhitespaces++class HasIndents s a | s -> a where+  -- | 'Traversal'' targeting the indentation inside a structure+  --+  -- Note: whitespace inside \'enclosed forms\' (such as lists or tuples) is not+  -- considered indentation.+  --+  -- e.g.+  --+  -- In the following code, there is only one chunk of indentation:+  --+  -- @+  -- def a():+  --     [ b+  --     , c+  --     , d+  --     ]+  -- @+  --+  -- it's here:+  --+  -- @+  -- def a():+  --     [ b+  -- ^^^^+  --     , c+  --     , d+  --     ]+  -- @+  --+  -- The rest is whitespace that is internal to the list.+  _Indents :: Traversal' s (Indents a)++instance HasIndents (PyToken a) a where+  _Indents f (TkIndent a i) = TkIndent a <$> f i+  _Indents f (TkLevel a i) = TkLevel a <$> f i+  _Indents _ a = pure a++instance HasIndents (Fundef '[] a) a where+  _Indents fun (MkFundef a b c d e f g h i j k) =+    (\b' c' -> MkFundef a b' c' d e f g h i j) <$>+    (traverse._Indents) fun b <*>+    fun c <*>+    _Indents fun k++instance HasIndents (For '[] a) a where+  _Indents fun (MkFor a b c d e f g h i) =+    (\b' -> MkFor a b' c d e f g) <$>+    fun b <*>+    _Indents fun h <*>+    (traverse._Indents) fun i++instance HasIndents (TryFinally '[] a) a where+  _Indents fun (MkTryFinally a b c d e) =+    (\b' -> MkTryFinally a b' c) <$>+    fun b <*>+    _Indents fun d <*>+    _Indents fun e++instance HasIndents (TryExcept '[] a) a where+  _Indents fun (MkTryExcept a b c d e f g) =+    (\b' -> MkTryExcept a b' c) <$>+    fun b <*>+    _Indents fun d <*>+    (traverse._Indents) fun e <*>+    (traverse._Indents) fun f <*>+    (traverse._Indents) fun g++instance HasIndents (Except '[] a) a where+  _Indents fun (MkExcept a b c d) =+    (\a' -> MkExcept a' b c) <$>+    fun a <*>+    _Indents fun d++instance HasIndents (Finally '[] a) a where+  _Indents fun (MkFinally a b c) =+    (\a' -> MkFinally a' b) <$>+    fun a <*>+    _Indents fun c++instance HasIndents (If '[] a) a where+  _Indents fun (MkIf a b c d e f g) =+    (\b' -> MkIf a b' c d) <$>+    fun b <*>+    _Indents fun e <*>+    (traverse._Indents) fun f <*>+    (traverse._Indents) fun g++instance HasIndents (While '[] a) a where+  _Indents fun (MkWhile a b c d e f) =+    (\b' -> MkWhile a b' c d) <$>+    fun b <*>+    _Indents fun e <*>+    (traverse._Indents) fun f++instance HasIndents (Elif '[] a) a where+  _Indents fun (MkElif a b c d) =+    (\a' -> MkElif a' b c) <$>+    fun a <*>+    _Indents fun d++instance HasIndents (Else '[] a) a where+  _Indents f (MkElse a b c) = MkElse <$> f a <*> pure b <*> _Indents f c++instance HasIndents (SmallStatement '[] a) a where+  _Indents _ (MkSmallStatement a b c d e) =+    pure $ MkSmallStatement a b c d e++instance HasIndents (Statement '[] a) a where+  _Indents f (SmallStatement idnt a) = SmallStatement <$> f idnt <*> _Indents f a+  _Indents f (CompoundStatement c) = CompoundStatement <$> _Indents f c++instance HasIndents (Block '[] a) a where+  _Indents = _Statements._Indents++instance HasIndents (Suite '[] a) a where+  _Indents _ (SuiteOne a b c) = pure $ SuiteOne a b c+  _Indents f (SuiteMany a b c d e) = SuiteMany a b c d <$> _Indents f e++instance HasIndents (Decorator '[] a) a where+  _Indents fun (Decorator a b c d e f g) =+    (\b' -> Decorator a b' c d e f g) <$>+    fun b++instance HasIndents (ClassDef '[] a) a where+  _Indents fun (MkClassDef a b c d e f g) =+    (\b' c' -> MkClassDef a b' c' d e f) <$>+    (traverse._Indents) fun b <*>+    fun c <*>+    _Indents fun g++instance HasIndents (With '[] a) a where+  _Indents fun (MkWith a b c d e f) =+    (\b' -> MkWith a b' c d e) <$>+    fun b <*>+    _Indents fun f++instance HasIndents (CompoundStatement '[] a) a where+  _Indents fun s =+    case s of+      Fundef a decos idnt asyncWs b c d e f g h ->+        (\decos' idnt' -> Fundef a decos' idnt' asyncWs b c d e f g) <$>+        (traverse._Indents) fun decos <*>+        fun idnt <*>+        _Indents fun h+      If a idnt b c d elifs e ->+        (\idnt' -> If a idnt' b c) <$>+        fun idnt <*>+        _Indents fun d <*>+        traverse+          (\(idnt, a, b, c) ->+             (\idnt'  -> (,,,) idnt' a b) <$>+             fun idnt <*>+             _Indents fun c)+          elifs <*>+        traverse+          (\(idnt, a, b) ->+             (\idnt' -> (,,) idnt' a) <$>+             fun idnt <*>+             _Indents fun b)+          e+      While a idnt b c d e ->+        (\idnt' -> While a idnt' b c) <$>+        fun idnt <*>+        _Indents fun d <*>+        traverse+          (\(idnt, a, b) ->+             (\idnt' -> (,,) idnt' a) <$>+             fun idnt <*>+             _Indents fun b)+          e+      TryExcept a idnt b c d e f ->+        (\idnt' -> TryExcept a idnt' b) <$>+        fun idnt <*>+        _Indents fun c <*>+        traverse+          (\(idnt, a, b, c) ->+             (\idnt' -> (,,,) idnt' a b) <$>+             fun idnt <*>+             _Indents fun c)+          d <*>+        traverse+          (\(idnt, a, b) ->+             (\idnt' -> (,,) idnt' a) <$>+             fun idnt <*>+             _Indents fun b)+          e <*>+        traverse+          (\(idnt, a, b) ->+             (\idnt' -> (,,) idnt' a) <$>+             fun idnt <*>+             _Indents fun b)+          f+      TryFinally a idnt b c idnt2 d e ->+        (\idnt' c' idnt2' -> TryFinally a idnt' b c' idnt2' d) <$>+        fun idnt <*>+        _Indents fun c <*>+        fun idnt2 <*>+        _Indents fun e+      For a idnt asyncWs b c d e f g ->+        (\idnt' -> For a idnt' asyncWs b c d e) <$>+        fun idnt <*>+        _Indents fun f <*>+        traverse+          (\(idnt, a, b) ->+             (\idnt' -> (,,) idnt' a) <$>+             fun idnt <*>+             _Indents fun b)+          g+      ClassDef a decos idnt b c d e ->+        (\decos' idnt' -> ClassDef a decos' idnt' b c d) <$>+        traverse (_Indents fun) decos <*>+        fun idnt <*>+        _Indents fun e+      With a b asyncWs c d e ->+        (\b' -> With a b' asyncWs c d) <$>+        fun b <*>+        _Indents fun e
+ src/Language/Python/Optics/Newlines.hs view
@@ -0,0 +1,576 @@+{-|+Module      : Language.Python.Optics.Newlines+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}+module Language.Python.Optics.Newlines where++import Control.Lens.Traversal (Traversal')+import Data.List.NonEmpty (NonEmpty(..))++import qualified Data.FingerTree as FingerTree++import Language.Python.Syntax++{-++I can't derive this with generic-lens :( it wants generic instances for certain+things that can't have those instance (and don't contain newlines anyways)++-}+class HasNewlines s where+  -- | 'Traversal'' targeting all of thie 'Newline's in a structure+  --+  -- This only targets places that contain the 'Newline' datatype; it doesn't target+  -- newline characters in string literals, for example.+  _Newlines :: Traversal' s Newline++instance (HasNewlines a, HasNewlines b) => HasNewlines (a, b) where+  _Newlines f (a, b) = (,) <$> _Newlines f a <*> _Newlines f b++instance (HasNewlines a, HasNewlines b, HasNewlines c) => HasNewlines (a, b, c) where+  _Newlines f (a, b, c) =+    (,,) <$>+    _Newlines f a <*>+    _Newlines f b <*>+    _Newlines f c++instance (HasNewlines a, HasNewlines b, HasNewlines c, HasNewlines d) => HasNewlines (a, b, c, d) where+  _Newlines f (a, b, c, d) =+    (,,,) <$>+    _Newlines f a <*>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines (e a) => HasNewlines (ImportAs e v a) where+  _Newlines f (ImportAs a b c) =+    ImportAs a <$>+    _Newlines f b <*>+    _Newlines f c++instance HasNewlines (RelativeModuleName v a) where+  _Newlines f (RelativeWithName a b) =+    RelativeWithName <$>+    _Newlines f a <*>+    _Newlines f b+  _Newlines f (Relative a) = Relative <$> _Newlines f a++instance (HasNewlines a, HasNewlines b) => HasNewlines (Either a b) where+  _Newlines f (Left a) = Left <$> _Newlines f a+  _Newlines f (Right a) = Right <$> _Newlines f a++instance HasNewlines Newline where+  _Newlines = id++instance HasNewlines a => HasNewlines [a] where+  _Newlines = traverse._Newlines++instance HasNewlines Whitespace where+  _Newlines _ Space = pure Space+  _Newlines _ Tab = pure Tab+  _Newlines f (Continued nl ws) = Continued <$> f nl <*> _Newlines f ws+  _Newlines _ (Comment c) = pure $ Comment c+  _Newlines f (Newline nl) = Newline <$> f nl++instance HasNewlines (Blank a) where+  _Newlines f (Blank a b c) = (\b' -> Blank a b' c) <$> _Newlines f b++instance HasNewlines (Block v a) where+  _Newlines f (Block a b c) =+    Block <$>+    _Newlines f a <*>+    _Newlines f b <*>+    _Newlines f c++instance HasNewlines Colon where+  _Newlines f (MkColon a) = MkColon <$> _Newlines f a++instance HasNewlines Dot where+  _Newlines f (MkDot a) = MkDot <$> _Newlines f a++instance HasNewlines Comma where+  _Newlines f (MkComma a) = MkComma <$> _Newlines f a++instance HasNewlines At where+  _Newlines f (MkAt a) = MkAt <$> _Newlines f a++instance HasNewlines (Semicolon a) where+  _Newlines f (MkSemicolon a b) = MkSemicolon a <$> _Newlines f b++instance HasNewlines Equals where+  _Newlines f (MkEquals a) = MkEquals <$> _Newlines f a++instance HasNewlines (Suite v a) where+  _Newlines f (SuiteOne a b c) = SuiteOne a b <$> _Newlines f c+  _Newlines f (SuiteMany a b c d e) =+    (\b' d' e' -> SuiteMany a b' c d' e') <$>+    _Newlines f b <*>+    f d <*>+    _Newlines f e++instance HasNewlines Indent where+  _Newlines f (MkIndent a) = MkIndent <$> (FingerTree.traverse'._Newlines) f a++instance HasNewlines (Indents a) where+  _Newlines f (Indents a b) = (\a' -> Indents a' b) <$> _Newlines f a++instance HasNewlines (UnOp a) where+  _Newlines f (Negate a b) = Negate a <$> _Newlines f b+  _Newlines f (Positive a b) = Positive a <$> _Newlines f b+  _Newlines f (Complement a b) = Complement a <$> _Newlines f b++instance HasNewlines (BinOp a) where+  _Newlines f x =+    case x of+      Is a b -> Is a <$> _Newlines f b+      IsNot a b c -> IsNot a <$> _Newlines f b <*> _Newlines f c+      In a b -> In a <$> _Newlines f b+      NotIn a b c -> NotIn a <$> _Newlines f b <*> _Newlines f c+      Minus a b -> Minus a <$> _Newlines f b+      Exp a b -> Exp a <$> _Newlines f b+      BoolAnd a b -> BoolAnd a <$> _Newlines f b+      BoolOr a b -> BoolOr a <$> _Newlines f b+      Eq a b -> Eq a <$> _Newlines f b+      Lt a b -> Lt a <$> _Newlines f b+      LtEq a b -> LtEq a <$> _Newlines f b+      Gt a b -> Gt a <$> _Newlines f b+      GtEq a b -> GtEq a <$> _Newlines f b+      NotEq a b -> NotEq a <$> _Newlines f b+      Multiply a b -> Multiply a <$> _Newlines f b+      Divide a b -> Divide a <$> _Newlines f b+      FloorDivide a b -> FloorDivide a <$> _Newlines f b+      Percent a b -> Percent a <$> _Newlines f b+      Plus a b -> Plus a <$> _Newlines f b+      BitOr a b -> BitOr a <$> _Newlines f b+      BitXor a b -> BitXor a <$> _Newlines f b+      BitAnd a b -> BitAnd a <$> _Newlines f b+      ShiftLeft a b -> ShiftLeft a <$> _Newlines f b+      ShiftRight a b -> ShiftRight a <$> _Newlines f b+      At a b -> At a <$> _Newlines f b++instance HasNewlines a => HasNewlines (CommaSep a) where+  _Newlines f = go+    where+      go CommaSepNone = pure CommaSepNone+      go (CommaSepOne a) = CommaSepOne <$> _Newlines f a+      go (CommaSepMany a b c) =+        CommaSepMany <$>+        _Newlines f a <*>+        _Newlines f b <*>+        go c++instance HasNewlines a => HasNewlines (CommaSep1 a) where+  _Newlines f = go+    where+      go (CommaSepOne1 a) = CommaSepOne1 <$> _Newlines f a+      go (CommaSepMany1 a b c) =+        CommaSepMany1 <$>+        _Newlines f a <*>+        _Newlines f b <*>+        go c++instance HasNewlines a => HasNewlines (CommaSep1' a) where+  _Newlines f = go+    where+      go (CommaSepOne1' a b) = CommaSepOne1' <$> _Newlines f a <*> _Newlines f b+      go (CommaSepMany1' a b c) =+        CommaSepMany1' <$>+        _Newlines f a <*>+        _Newlines f b <*>+        go c++instance HasNewlines (Ident v a) where+  _Newlines f (MkIdent a b c) = MkIdent a b <$> _Newlines f c++instance HasNewlines a => HasNewlines (Maybe a) where+  _Newlines = traverse._Newlines++instance HasNewlines (Param v a) where+  _Newlines f (PositionalParam a b c) =+    PositionalParam a <$>+    _Newlines f b <*>+    _Newlines f c+  _Newlines f (KeywordParam a b c d e) =+    KeywordParam a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d <*>+    _Newlines f e+  _Newlines f (StarParam a b c d) =+    StarParam a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d+  _Newlines f (UnnamedStarParam a b) =+    UnnamedStarParam a <$>+    _Newlines f b+  _Newlines f (DoubleStarParam a b c d) =+    DoubleStarParam a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines (Arg v a) where+  _Newlines f (PositionalArg a b) =+    PositionalArg a <$>+    _Newlines f b+  _Newlines f (KeywordArg a b c d) =+    KeywordArg a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d+  _Newlines f (StarArg a b c) =+    StarArg a <$>+    _Newlines f b <*>+    _Newlines f c+  _Newlines f (DoubleStarArg a b c) =+    DoubleStarArg a <$>+    _Newlines f b <*>+    _Newlines f c++instance HasNewlines (CompFor v a) where+  _Newlines f (CompFor a b c d e) =+    CompFor a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d <*>+    _Newlines f e++instance HasNewlines (CompIf v a) where+  _Newlines f (CompIf a b c) =+    CompIf a <$>+    _Newlines f b <*>+    _Newlines f c++instance HasNewlines (e v a) => HasNewlines (Comprehension e v a) where+  _Newlines f (Comprehension a b c d) =+    Comprehension a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines (TupleItem v a) where+  _Newlines f (TupleItem a b) = TupleItem a <$> _Newlines f b+  _Newlines f (TupleUnpack a b c d) =+    TupleUnpack a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines (ListItem v a) where+  _Newlines f (ListItem a b) = ListItem a <$> _Newlines f b+  _Newlines f (ListUnpack a b c d) =+    ListUnpack a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines (SetItem v a) where+  _Newlines f (SetItem a b) = SetItem a <$> _Newlines f b+  _Newlines f (SetUnpack a b c d) =+    SetUnpack a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines (DictItem v a) where+  _Newlines f (DictItem a b c d) =+    DictItem a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d+  _Newlines f (DictUnpack a b c) =+    DictUnpack a <$>+    _Newlines f b <*>+    _Newlines f c++instance HasNewlines (Subscript v a) where+  _Newlines f (SubscriptExpr a) = SubscriptExpr <$> _Newlines f a+  _Newlines f (SubscriptSlice a b c d) =+    SubscriptSlice <$>+    _Newlines f a <*>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines a => HasNewlines (NonEmpty a) where+  _Newlines f (a :| as) = (:|) <$> _Newlines f a <*> _Newlines f as++instance HasNewlines (StringLiteral a) where+  _Newlines = stringLiteralWhitespace.traverse._Newlines++instance HasNewlines (Expr v a) where+  _Newlines fun = go+    where+      go e =+        case e of+          Unit a b c -> Unit a <$> _Newlines fun b <*> _Newlines fun c+          Lambda a b c d e ->+            Lambda a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d <*>+            go e+          Yield a b c ->+            Yield a <$> _Newlines fun b <*> _Newlines fun c+          YieldFrom a b c d ->+            YieldFrom a <$> _Newlines fun b <*> _Newlines fun c <*> go d+          Ternary a b c d e f ->+            Ternary a <$>+            go b <*>+            _Newlines fun c <*>+            go d <*>+            _Newlines fun e <*>+            go f+          ListComp a b c d ->+            ListComp a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d+          List a b c d ->+            List a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d+          DictComp a b c d ->+            DictComp a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d+          Dict a b c d ->+            Dict a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d+          SetComp a b c d ->+            SetComp a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d+          Set a b c d ->+            Set a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d+          Deref a b c d ->+            Deref a <$>+            go b <*>+            _Newlines fun c <*>+            _Newlines fun d+          Subscript a b c d e ->+            Subscript a <$>+            go b <*>+            _Newlines fun c <*>+            _Newlines fun d <*>+            _Newlines fun e+          Call a b c d e ->+            Call a <$>+            go b <*>+            _Newlines fun c <*>+            _Newlines fun d <*>+            _Newlines fun e+          None a b -> None a <$> _Newlines fun b+          Ellipsis a b -> Ellipsis a <$> _Newlines fun b+          BinOp a b c d ->+            BinOp a <$>+            go b <*>+            _Newlines fun c <*>+            go d+          UnOp a b c ->+            UnOp a <$>+            _Newlines fun b <*>+            go c+          Parens a b c d ->+            Parens a <$>+            _Newlines fun b <*>+            go c <*>+            _Newlines fun d+          Ident a -> Ident <$> _Newlines fun a+          Int a b c -> Int a b <$> _Newlines fun c+          Float a b c -> Float a b <$> _Newlines fun c+          Imag a b c -> Imag a b <$> _Newlines fun c+          Bool a b c -> Bool a b <$> _Newlines fun c+          String a b -> String a <$> _Newlines fun b+          Tuple a b c d ->+            Tuple a <$>+            _Newlines fun b <*>+            _Newlines fun c <*>+            _Newlines fun d+          Not a b c -> Not a <$> _Newlines fun b <*> go c+          Generator a b -> Generator a <$> _Newlines fun b+          Await a b c -> Await a <$> _Newlines fun b <*> _Newlines fun c++instance HasNewlines (Decorator v a) where+  _Newlines fun (Decorator a b c d e f g) =+    Decorator a <$>+    _Newlines fun b <*>+    _Newlines fun c <*>+    _Newlines fun d <*>+    pure e <*>+    fun f <*>+    _Newlines fun g++instance HasNewlines (ExceptAs v a) where+  _Newlines f (ExceptAs a b c) = ExceptAs a <$> _Newlines f b <*> _Newlines f c++instance HasNewlines (WithItem v a) where+  _Newlines f (WithItem a b c) = WithItem a <$> _Newlines f b <*> _Newlines f c++instance HasNewlines (CompoundStatement v a) where+  _Newlines fun s =+    case s of+      Fundef ann decos idnt asyncWs ws1 name ws2 params ws3 mty s ->+        Fundef ann <$>+        _Newlines fun decos <*>+        _Newlines fun idnt <*>+        _Newlines fun asyncWs <*>+        _Newlines fun ws1 <*>+        _Newlines fun name <*>+        _Newlines fun ws2 <*>+        _Newlines fun params <*>+        _Newlines fun ws3 <*>+        _Newlines fun mty <*>+        _Newlines fun s+      If ann idnt ws1 cond s elifs els ->+        If ann <$>+        _Newlines fun idnt <*>+        _Newlines fun ws1 <*>+        _Newlines fun cond <*>+        _Newlines fun s <*>+        _Newlines fun elifs <*>+        _Newlines fun els+      While ann idnt ws1 cond s els ->+        While ann <$>+        _Newlines fun idnt <*>+        _Newlines fun ws1 <*>+        _Newlines fun cond <*>+        _Newlines fun s <*>+        _Newlines fun els+      TryExcept ann idnt b c f k l ->+        TryExcept ann <$>+        _Newlines fun idnt <*>+        _Newlines fun b <*>+        _Newlines fun c <*>+        _Newlines fun f <*>+        _Newlines fun k <*>+        _Newlines fun l+      TryFinally ann idnt b c idnt2 f g ->+        TryFinally ann <$>+        _Newlines fun idnt <*>+        _Newlines fun b <*>+        _Newlines fun c <*>+        _Newlines fun idnt2 <*>+        _Newlines fun f <*>+        _Newlines fun g+      For ann idnt asyncWs b c d e f g ->+        For ann <$>+        _Newlines fun idnt <*>+        _Newlines fun asyncWs <*>+        _Newlines fun b <*>+        _Newlines fun c <*>+        _Newlines fun d <*>+        _Newlines fun e <*>+        _Newlines fun f <*>+        _Newlines fun g+      ClassDef a decos idnt b c d e ->+        ClassDef a <$>+        _Newlines fun decos <*>+        _Newlines fun idnt <*>+        _Newlines fun b <*>+        _Newlines fun c <*>+        _Newlines fun d <*>+        _Newlines fun e+      With a b asyncWs c d e ->+        With a <$>+        _Newlines fun b <*>+        _Newlines fun asyncWs <*>+        _Newlines fun c <*>+        _Newlines fun d <*>+        _Newlines fun e++instance HasNewlines (ModuleName v a) where+  _Newlines f = go+    where+      go (ModuleNameOne a b) =+        ModuleNameOne a <$> _Newlines f b+      go (ModuleNameMany a b c d) =+        ModuleNameMany a <$> _Newlines f b <*> _Newlines f c <*> go d++instance HasNewlines (ImportTargets v a) where+  _Newlines f (ImportAll a b) =+    ImportAll a <$> _Newlines f b+  _Newlines f (ImportSome a b) =+    ImportSome a <$> _Newlines f b+  _Newlines f (ImportSomeParens a b c d) =+    ImportSomeParens a <$>+    _Newlines f b <*>+    _Newlines f c <*>+    _Newlines f d++instance HasNewlines (SimpleStatement v a) where+  _Newlines fun s =+    case s of+      Return a b c -> Return a <$> _Newlines fun b <*> _Newlines fun c+      Expr a b -> Expr a <$> _Newlines fun b+      Assign a b c -> Assign a <$> _Newlines fun b <*> _Newlines fun c+      AugAssign a b c d ->+        AugAssign a <$>+        _Newlines fun b <*>+        pure c <*>+        _Newlines fun d+      Pass a b -> Pass a <$> _Newlines fun b+      Break a b -> Break a <$> _Newlines fun b+      Continue a b -> Continue a <$> _Newlines fun b+      Global a b c -> Global a <$> _Newlines fun b <*> _Newlines fun c+      Nonlocal a b c -> Nonlocal a <$> _Newlines fun b <*> _Newlines fun c+      Del a b c -> Del a <$> _Newlines fun b <*> _Newlines fun c+      Import a b c ->+        Import a <$>+        _Newlines fun b <*>+        _Newlines fun c+      From a b c d e ->+        From a <$>+        _Newlines fun b <*>+        _Newlines fun c <*>+        _Newlines fun d <*>+        _Newlines fun e+      Raise a b c ->+        Raise a <$>+        _Newlines fun b <*>+        _Newlines fun c+      Assert a b c d ->+        Assert a <$>+        _Newlines fun b <*>+        _Newlines fun c <*>+        _Newlines fun d++instance HasNewlines (SmallStatement v a) where+  _Newlines f (MkSmallStatement s ss sc cmt nl) =+    MkSmallStatement <$>+    _Newlines f s <*>+    _Newlines f ss <*>+    _Newlines f sc <*>+    pure cmt <*>+    _Newlines f nl++instance HasNewlines (Statement v a) where+  _Newlines f (CompoundStatement c) =+    CompoundStatement <$> _Newlines f c+  _Newlines f (SmallStatement i a) =+    SmallStatement <$>+    _Newlines f i <*>+    _Newlines f a++instance HasNewlines (Module v a) where+  _Newlines f = go+    where+      go ModuleEmpty = pure ModuleEmpty+      go (ModuleBlankFinal a) = pure $ ModuleBlankFinal a+      go (ModuleBlank a b c) =+        ModuleBlank a <$> f b <*> go c+      go (ModuleStatement a b) =+        ModuleStatement <$> _Newlines f a <*> go b
+ src/Language/Python/Optics/Validated.hs view
@@ -0,0 +1,25 @@+{-# language DataKinds, PolyKinds, DefaultSignatures #-}++{-|+Module      : Language.Python.Optics.Validated+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Optics.Validated (+  Validated (unvalidated)+) where++import Control.Lens.Getter (Getter, to)+import Data.Coerce (Coercible, coerce)++-- | A type class for things for which we can strip the validation information.+-- This can help types line up when they need to, for example to put many+-- things of various validation statuses together in a list.+class Validated (s :: [*] -> * -> *) where+  unvalidated :: Getter (s v a) (s '[] a)+  default unvalidated :: Coercible (s v a) (s '[] a) => Getter (s v a) (s '[] a)+  unvalidated = to coerce
+ src/Language/Python/Parse.hs view
@@ -0,0 +1,143 @@+{-# language DataKinds #-}+{-# language FlexibleContexts #-}+{-# language MultiParamTypeClasses, FlexibleInstances #-}++{-|+Module      : Language.Python.Parse+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Parse+  ( module Language.Python.Parse.Error+  , Parser+  , parseModule+  , parseStatement+  , parseExpr+  , parseExprList+    -- * Source Information+  , SrcInfo(..), initialSrcInfo+  )+where++import Control.Applicative ((<|>))+import Data.Bifunctor (first)+import Data.List.NonEmpty (NonEmpty)+import Data.Text (Text)+import Data.Validation (Validation, bindValidation, fromEither)+import Text.Megaparsec (eof)++import Language.Python.Internal.Lexer+  ( SrcInfo(..), initialSrcInfo, withSrcInfo+  , tokenize, insertTabs+  )+import Language.Python.Internal.Token (PyToken)+import Language.Python.Internal.Parse+  ( Parser, runParser, level, module_, statement, exprOrStarList+  , expr, space+  )+import Language.Python.Internal.Syntax.IR (AsIRError)+import Language.Python.Parse.Error+import Language.Python.Syntax.Expr (Expr)+import Language.Python.Syntax.Module (Module)+import Language.Python.Syntax.Statement (Statement)+import Language.Python.Syntax.Whitespace (Indents (..))++import qualified Language.Python.Internal.Syntax.IR as IR++-- | Parse a module+--+-- https://docs.python.org/3/reference/toplevel_components.html#file-input+parseModule+  :: ( AsLexicalError e Char+     , AsTabError e SrcInfo+     , AsIncorrectDedent e SrcInfo+     , AsParseError e (PyToken SrcInfo)+     , AsIRError e SrcInfo+     )+  => FilePath -- ^ File name+  -> Text -- ^ Input to parse+  -> Validation (NonEmpty e) (Module '[] SrcInfo)+parseModule fp input =+  let+    si = initialSrcInfo fp+    ir = do+      tokens <- tokenize fp input+      tabbed <- insertTabs si tokens+      runParser fp module_ tabbed+  in+    fromEither (first pure ir) `bindValidation` IR.fromIR++-- | Parse a statement+--+-- https://docs.python.org/3/reference/compound_stmts.html#grammar-token-statement+parseStatement+  :: ( AsLexicalError e Char+     , AsTabError e SrcInfo+     , AsIncorrectDedent e SrcInfo+     , AsParseError e (PyToken SrcInfo)+     , AsIRError e SrcInfo+     )+  => FilePath -- ^ File name+  -> Text -- ^ Input to parse+  -> Validation (NonEmpty e) (Statement '[] SrcInfo)+parseStatement fp input =+  let+    si = initialSrcInfo fp+    ir = do+      tokens <- tokenize fp input+      tabbed <- insertTabs si tokens+      runParser fp ((statement tlIndent =<< tlIndent) <* eof) tabbed+  in+    fromEither (first pure ir) `bindValidation` IR.fromIR_statement+  where+    tlIndent = level <|> withSrcInfo (pure $ Indents [])++-- | Parse an expression list (unparenthesised tuple)+--+-- https://docs.python.org/3.5/reference/expressions.html#grammar-token-expression_list+parseExprList+  :: ( AsLexicalError e Char+     , AsTabError e SrcInfo+     , AsIncorrectDedent e SrcInfo+     , AsParseError e (PyToken SrcInfo)+     , AsIRError e SrcInfo+     )+  => FilePath -- ^ File name+  -> Text -- ^ Input to parse+  -> Validation (NonEmpty e) (Expr '[] SrcInfo)+parseExprList fp input =+  let+    si = initialSrcInfo fp+    ir = do+      tokens <- tokenize fp input+      tabbed <- insertTabs si tokens+      runParser fp (exprOrStarList space <* eof) tabbed+  in+    fromEither (first pure ir) `bindValidation` IR.fromIR_expr++-- | Parse an expression+--+-- https://docs.python.org/3.5/reference/expressions.html#grammar-token-expression+parseExpr+  :: ( AsLexicalError e Char+     , AsTabError e SrcInfo+     , AsIncorrectDedent e SrcInfo+     , AsParseError e (PyToken SrcInfo)+     , AsIRError e SrcInfo+     )+  => FilePath -- ^ File name+  -> Text -- ^ Input to parse+  -> Validation (NonEmpty e) (Expr '[] SrcInfo)+parseExpr fp input =+  let+    si = initialSrcInfo fp+    ir = do+      tokens <- tokenize fp input+      tabbed <- insertTabs si tokens+      runParser fp (expr space <* eof) tabbed+  in+    fromEither (first pure ir) `bindValidation` IR.fromIR_expr
+ src/Language/Python/Parse/Error.hs view
@@ -0,0 +1,96 @@+{-# language FlexibleInstances, MultiParamTypeClasses #-}+{-# language LambdaCase #-}+module Language.Python.Parse.Error+  ( ParseError(..)+    -- * Classy Prisms+  , AsLexicalError(..), AsTabError(..), AsIncorrectDedent(..)+  , AsIRError(..), AsParseError(..)+    -- * Megaparsec re-exports+  , ErrorItem(..)+  , SourcePos(..)+  )+where++import Control.Lens.Prism (prism')+import Data.Set (Set)+import Data.List.NonEmpty (NonEmpty)+import Text.Megaparsec.Error (ErrorItem(..))+import Text.Megaparsec.Pos (SourcePos(..))++import Language.Python.Internal.Lexer+  (AsLexicalError(..), AsTabError(..), AsIncorrectDedent(..))+import Language.Python.Internal.Parse (AsParseError(..))+import Language.Python.Internal.Syntax.IR (AsIRError(..))+import Language.Python.Internal.Token (PyToken)++data ParseError a+  -- | An error occured during tokenization (this is a re-packed megaparsec error)+  = LexicalError+      (NonEmpty SourcePos)+      (Maybe (ErrorItem Char))+      (Set (ErrorItem Char))+  -- | An error occured during parsing (this is a re-packed megaparsec error)+  | ParseError+      (NonEmpty SourcePos)+      (Maybe (ErrorItem (PyToken a)))+      (Set (ErrorItem (PyToken a)))+  -- | Tabs and spaces were used inconsistently+  | TabError a+  -- | The dedent at the end of a block doesn't match and preceding indents+  --+  -- e.g.+  --+  -- @+  -- def a():+  --     if b:+  --         pass+  --     else:+  --         pass+  --   pass+  -- @+  --+  -- The final line will cause an 'IncorrectDedent' error+  | IncorrectDedent a+  -- | Unpacking ( @*value@ ) was used in an invalid position+  | InvalidUnpacking a+  deriving (Eq, Show)++instance AsLexicalError (ParseError a) Char where+  _LexicalError =+    prism'+      (\(a, b, c) -> LexicalError a b c)+      (\case+          LexicalError a b c -> Just (a, b ,c)+          _ -> Nothing)++instance AsTabError (ParseError a) a where+  _TabError =+    prism'+      TabError+      (\case+          TabError a -> Just a+          _ -> Nothing)++instance AsIncorrectDedent (ParseError a) a where+  _IncorrectDedent =+    prism'+      IncorrectDedent+      (\case+          IncorrectDedent a -> Just a+          _ -> Nothing)++instance AsParseError (ParseError a) (PyToken a) where+  _ParseError =+    prism'+      (\(a, b, c) -> ParseError a b c)+      (\case+          ParseError a b c -> Just (a, b ,c)+          _ -> Nothing)++instance AsIRError (ParseError a) a where+  _InvalidUnpacking =+    prism'+      InvalidUnpacking+      (\case+          InvalidUnpacking a -> Just a+          _ -> Nothing)
+ src/Language/Python/Render.hs view
@@ -0,0 +1,15 @@+{-|+Module      : Language.Python.Render+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Render+  ( showModule, showStatement, showExpr+  )+where++import Language.Python.Internal.Render
+ src/Language/Python/Syntax.hs view
@@ -0,0 +1,49 @@+{-|+Module      : Language.Python.Syntax+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++The abstract syntax tree for Python.++Key datatypes include 'Module', 'Statement', and 'Expr'.+-}++module Language.Python.Syntax+  ( module Language.Python.Syntax.AugAssign+  , module Language.Python.Syntax.CommaSep+  , module Language.Python.Syntax.Comment+  , module Language.Python.Syntax.Expr+  , module Language.Python.Syntax.Ident+  , module Language.Python.Syntax.Import+  , module Language.Python.Syntax.Module+  , module Language.Python.Syntax.ModuleNames+  , module Language.Python.Syntax.Numbers+  , module Language.Python.Syntax.Operator.Binary+  , module Language.Python.Syntax.Operator.Unary+  , module Language.Python.Syntax.Punctuation+  , module Language.Python.Syntax.Statement+  , module Language.Python.Syntax.Strings+  , module Language.Python.Syntax.Types+  , module Language.Python.Syntax.Whitespace+  )+where++import Language.Python.Syntax.AugAssign+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Comment+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Import+import Language.Python.Syntax.Module+import Language.Python.Syntax.ModuleNames+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Operator.Binary+import Language.Python.Syntax.Operator.Unary+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Types+import Language.Python.Syntax.Whitespace
+ src/Language/Python/Syntax/AugAssign.hs view
@@ -0,0 +1,72 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}++{-|+Module      : Language.Python.Syntax.AugAssign+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.AugAssign where++import Control.Lens.Lens (lens)++import Language.Python.Syntax.Whitespace++-- | Augmented assignments (PEP 203), such as:+--+-- @+-- x += y+-- @+--+-- or+--+-- @+-- x <<= 8+-- @+--+-- An 'AugAssign' has an 'AugAssignOp' and trailing whitespace. There is an+-- optional annotation, which can simply be @()@ if no annotation is desired.+data AugAssign a+  = MkAugAssign+  { _augAssignType :: AugAssignOp+  , _augAssignAnn :: a+  , _augAssignWhitespace :: [Whitespace]+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (AugAssign a) where+  trailingWhitespace =+    lens _augAssignWhitespace (\a b -> a { _augAssignWhitespace = b })++-- | Augmented assignment operators+data AugAssignOp+  -- | @+=@+  = PlusEq+  -- | @-=@+  | MinusEq+  -- | @*=@+  | StarEq+  -- | @\@=@+  | AtEq+  -- | @/=@+  | SlashEq+  -- | @%=@+  | PercentEq+  -- | @&=@+  | AmpersandEq+  -- | @|=@+  | PipeEq+  -- | @^=@+  | CaretEq+  -- | @<<=@+  | ShiftLeftEq+  -- | @>>=@+  | ShiftRightEq+  -- | @**=@+  | DoubleStarEq+  -- | @//=@+  | DoubleSlashEq+  deriving (Eq, Show)
+ src/Language/Python/Syntax/CommaSep.hs view
@@ -0,0 +1,210 @@+{-# language LambdaCase #-}+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}++{-|+Module      : Language.Python.Syntax.CommaSep+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.CommaSep+  ( Comma(..)+  , CommaSep(..), _CommaSep, csTrailingWhitespace+  , appendCommaSep, maybeToCommaSep, listToCommaSep+  , CommaSep1(..)+  , commaSep1Head, appendCommaSep1, listToCommaSep1, listToCommaSep1'+  , CommaSep1'(..)+  , _CommaSep1'+  )+where++import Control.Lens.Getter ((^.))+import Control.Lens.Iso (Iso, iso)+import Control.Lens.Lens (lens)+import Control.Lens.Setter ((.~))+import Control.Lens.Traversal (Traversal')+import Data.Coerce (coerce)+import Data.Function ((&))+import Data.Functor (($>))+import Data.List.NonEmpty (NonEmpty(..))+import Data.Maybe (fromMaybe)+import Data.Semigroup (Semigroup(..))++import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Whitespace (Whitespace (Space), HasTrailingWhitespace (..))++-- | Items separated by commas, with optional whitespace following each comma+data CommaSep a+  = CommaSepNone+  | CommaSepOne a+  | CommaSepMany a Comma (CommaSep a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | 'Traversal' targeting the trailing whitespace in a comma separated list.+--+-- This can't be an instance of 'HasTrailingWhitespace' because 'CommaSepNone' never+-- has trailing whitespace.+csTrailingWhitespace+  :: HasTrailingWhitespace a+  => Traversal' (CommaSep a) [Whitespace]+csTrailingWhitespace _ CommaSepNone = pure CommaSepNone+csTrailingWhitespace f (CommaSepOne a) = CommaSepOne <$> trailingWhitespace f a+csTrailingWhitespace f (CommaSepMany a (MkComma b) CommaSepNone) =+  (\b' -> CommaSepMany a (MkComma b') CommaSepNone) <$> f b+csTrailingWhitespace f (CommaSepMany a b c) =+  CommaSepMany a b <$> csTrailingWhitespace f c+++-- | Convert a maybe to a singleton or nullary 'CommaSep'+maybeToCommaSep :: Maybe a -> CommaSep a+maybeToCommaSep = maybe CommaSepNone CommaSepOne++-- | Convert a list to a 'CommaSep'+--+-- Anywhere where whitespace is ambiguous, this function puts a single space+listToCommaSep :: [a] -> CommaSep a+listToCommaSep [] = CommaSepNone+listToCommaSep [a] = CommaSepOne a+listToCommaSep (a:as) = CommaSepMany a (MkComma [Space]) $ listToCommaSep as++-- | Appends two comma separated values together.+--+-- The provided whitespace is to follow the joining comma which is added+appendCommaSep :: [Whitespace] -> CommaSep a -> CommaSep a -> CommaSep a+appendCommaSep _  CommaSepNone b = b+appendCommaSep _  (CommaSepOne a) CommaSepNone = CommaSepOne a+appendCommaSep ws (CommaSepOne a) (CommaSepOne b) = CommaSepMany a (MkComma ws) (CommaSepOne b)+appendCommaSep ws (CommaSepOne a) (CommaSepMany b c cs) = CommaSepMany a (MkComma ws) (CommaSepMany b c cs)+appendCommaSep ws (CommaSepMany a c cs) b = CommaSepMany a c (appendCommaSep ws cs b)++instance Semigroup (CommaSep a) where+  (<>) = appendCommaSep [Space]++instance Monoid (CommaSep a) where+  mempty  = CommaSepNone+  mappend = (<>)++-- | Non-empty 'CommaSep'+data CommaSep1 a+  = CommaSepOne1 a+  | CommaSepMany1 a Comma (CommaSep1 a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | Get the first element of a 'CommaSep1'+commaSep1Head :: CommaSep1 a -> a+commaSep1Head (CommaSepOne1 a) = a+commaSep1Head (CommaSepMany1 a _ _) = a++-- | Appends two non-empty comma separated values together.+--+-- The provided whitespace is to follow the joining comma which is added+appendCommaSep1 :: [Whitespace] -> CommaSep1 a -> CommaSep1 a -> CommaSep1 a+appendCommaSep1 ws a b =+  CommaSepMany1+    (case a of; CommaSepOne1 x -> x;  CommaSepMany1 x _ _  -> x)+    (case a of; CommaSepOne1 _ -> MkComma ws; CommaSepMany1 _ ws' _ -> ws')+    (case a of; CommaSepOne1 _ -> b;  CommaSepMany1 _ _ x  -> x <> b)++instance Semigroup (CommaSep1 a) where+  (<>) = appendCommaSep1 [Space]++instance HasTrailingWhitespace s => HasTrailingWhitespace (CommaSep1 s) where+  trailingWhitespace =+    lens+      (\case+         CommaSepOne1 a -> a ^. trailingWhitespace+         CommaSepMany1 _ _ a -> a ^. trailingWhitespace)+      (\cs ws ->+         case cs of+           CommaSepOne1 a ->+             CommaSepOne1 (a & trailingWhitespace .~ ws)+           CommaSepMany1 a b c -> CommaSepMany1 (coerce a) b (c & trailingWhitespace .~ ws))++-- | Convert a 'NonEmpty' to a 'CommaSep1'+--+-- Anywhere where whitespace is ambiguous, this function puts a single space+listToCommaSep1 :: NonEmpty a -> CommaSep1 a+listToCommaSep1 (a :| as) = go (a:as)+  where+    go [] = error "impossible"+    go [x] = CommaSepOne1 x+    go (x:xs) = CommaSepMany1 x (MkComma [Space]) $ go xs++-- | Non-empty 'CommaSep', optionally terminated by a comma+--+-- Assumes that the contents consumes trailing whitespace+data CommaSep1' a+  = CommaSepOne1' a (Maybe Comma)+  | CommaSepMany1' a Comma (CommaSep1' a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | Iso to unpack a 'CommaSep'+_CommaSep+  :: Iso+       (Maybe (a, [(Comma, a)], Maybe Comma))+       (Maybe (b, [(Comma, b)], Maybe Comma))+       (CommaSep a)+       (CommaSep b)+_CommaSep = iso toCs fromCs+  where+    toCs :: Maybe (a, [(Comma, a)], Maybe Comma) -> CommaSep a+    toCs Nothing = CommaSepNone+    toCs (Just (a, b, c)) =+      case b of+        [] -> maybe (CommaSepOne a) (\c' -> CommaSepMany a c' CommaSepNone) c+        (d, e):ds -> CommaSepMany a d $ toCs (Just (e, ds, c))++    fromCs :: CommaSep a -> Maybe (a, [(Comma, a)], Maybe Comma)+    fromCs CommaSepNone = Nothing+    fromCs (CommaSepOne a) = Just (a, [], Nothing)+    fromCs (CommaSepMany a b c) =+      case fromCs c of+        Nothing -> Just (a, [], Just b)+        Just (x, y, z) -> Just (a, (b, x) : y, z)++-- | Iso to unpack a 'CommaSep1''+_CommaSep1'+  :: Iso+       (a, [(Comma, a)], Maybe Comma)+       (b, [(Comma, b)], Maybe Comma)+       (CommaSep1' a)+       (CommaSep1' b)+_CommaSep1' = iso toCs fromCs+  where+    toCs (a, [], b) = CommaSepOne1' a b+    toCs (a, (b, c) : bs, d) = CommaSepMany1' a b $ toCs (c, bs, d)++    fromCs (CommaSepOne1' a b) = (a, [], b)+    fromCs (CommaSepMany1' a b c) =+      let+        (d, e, f) = fromCs c+      in+        (a, (b, d) : e, f)++-- | Attempt to insert comma separators into a list, which will not be+-- terminated by a comma.+--+-- If the list is empty, 'Nothing' is returned.+listToCommaSep1' :: [a] -> Maybe (CommaSep1' a)+listToCommaSep1' [] = Nothing+listToCommaSep1' [a] = Just (CommaSepOne1' a Nothing)+listToCommaSep1' (a:as) =+  CommaSepMany1' a (MkComma [Space]) <$> listToCommaSep1' as++instance HasTrailingWhitespace s => HasTrailingWhitespace (CommaSep1' s) where+  trailingWhitespace =+    lens+      (\case+         CommaSepOne1' a b -> maybe (a ^. trailingWhitespace) (^. trailingWhitespace) b+         CommaSepMany1' _ _ a -> a ^. trailingWhitespace)+      (\cs ws ->+         case cs of+           CommaSepOne1' a b ->+             CommaSepOne1'+               (fromMaybe (a & trailingWhitespace .~ ws) $ b $> coerce a)+               (b $> MkComma ws)+           CommaSepMany1' a b c ->+             CommaSepMany1' (coerce a) b (c & trailingWhitespace .~ ws))
+ src/Language/Python/Syntax/Comment.hs view
@@ -0,0 +1,44 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Syntax.Comment+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.Comment where++import Data.Deriving (deriveEq1, deriveOrd1)++-- | A Python single-line comment, such as on the following line:+--+-- @+-- y = x + 4 # add four to the value of x+-- @+--+-- In this case, the structure parsed would be+--+-- @+-- MkComment () " add four to the value of x"+-- @+--+-- with the hash being inferred, and the space after the hash being preserved.+--+-- Python does not have multi-line comments. There is a common convention of+-- using a multi-line string expression as a multi-line comment, since a+-- string expression is a no-op statement. Such multi-line comments are+-- __NOT__ represented with this data type, but rather as normal+-- string expressions (since that's what they are).+data Comment a+  = MkComment+  { _commentAnn :: a+  , _commentValue :: String+  }+  deriving (Eq, Ord, Show, Functor, Foldable, Traversable)++deriveEq1 ''Comment+deriveOrd1 ''Comment
+ src/Language/Python/Syntax/Expr.hs view
@@ -0,0 +1,1096 @@+{-# language LambdaCase #-}+{-# language DataKinds, KindSignatures #-}+{-# language ScopedTypeVariables #-}+{-# language MultiParamTypeClasses, FlexibleInstances #-}+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable, DeriveGeneric #-}+{-# language ExistentialQuantification #-}++{-|+Module      : Language.Python.Syntax.Expr+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.Expr+  ( -- * Expressions+    Expr(..), HasExprs(..), exprAnn, shouldGroupLeft, shouldGroupRight+    -- * Parameters and arguments+  , Param(..), paramAnn, paramType_, paramType, paramName+  , Arg(..), argExpr+    -- * Comprehension expressions+    -- | https://docs.python.org/3/reference/expressions.html#grammar-token-comprehension+  , Comprehension(..), CompIf(..), CompFor(..)+    -- * Collection items+  , DictItem(..), ListItem(..), SetItem(..), TupleItem(..)+    -- * Subscripts+  , Subscript(..)+  )+where++import Control.Lens.Cons (_last)+import Control.Lens.Fold ((^?), (^?!))+import Control.Lens.Getter ((^.), getting, to, view)+import Control.Lens.Lens (Lens, Lens', lens)+import Control.Lens.Plated (Plated(..), gplate)+import Control.Lens.Prism (_Just, _Left, _Right)+import Control.Lens.Setter ((.~), mapped, over)+import Control.Lens.Traversal (Traversal, failing, traverseOf)+import Control.Lens.Tuple (_2)+import Data.Bifunctor (bimap)+import Data.Bifoldable (bifoldMap)+import Data.Bitraversable (bitraverse)+import Data.Coerce (coerce)+import Data.Digit.Integral (integralDecDigits)+import Data.Function ((&))+import Data.List.NonEmpty (NonEmpty)+import Data.Maybe (isNothing)+import Data.Monoid ((<>))+import Data.String (IsString(..))+import GHC.Generics (Generic)+import Unsafe.Coerce (unsafeCoerce)++import Language.Python.Optics.Validated (Validated(..))+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Numbers+import Language.Python.Syntax.Operator.Binary+import Language.Python.Syntax.Operator.Unary+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace++{-++[unsafeCoerce Validation]++We can't 'coerce' 'Expr's because the @v@ parameter is considered to have a+nominal role, due to datatypes like 'Comprehension'. We only ever use @v@ in+as a phantom in 'Expr', so 'unsafeCoerce :: Expr v a -> Expr '[] a' is safe.++-}+instance Validated Expr where; unvalidated = to unsafeCoerce+instance Validated Param where; unvalidated = to unsafeCoerce+instance Validated Arg where; unvalidated = to unsafeCoerce+instance Validated DictItem where; unvalidated = to unsafeCoerce+instance Validated SetItem where; unvalidated = to unsafeCoerce+instance Validated TupleItem where; unvalidated = to unsafeCoerce+instance Validated ListItem where; unvalidated = to unsafeCoerce++-- | 'Control.Lens.Traversal.Traversal' over all the expressions in a term+class HasExprs s where+  _Exprs :: Traversal (s v a) (s '[] a) (Expr v a) (Expr '[] a)++-- | Formal parameters for functions+--+-- See <https://docs.python.org/3.5/reference/compound_stmts.html#function-definitions>+data Param (v :: [*]) a+  -- | @def foo(a):@+  = PositionalParam+  { _paramAnn :: a+  , _paramName :: Ident v a+  , _paramType :: Maybe (Colon, Expr v a)+  }+  -- | @def foo(bar=None):@+  | KeywordParam+  { _paramAnn :: a+  , _paramName :: Ident v a+  -- ':' spaces <expr>+  , _paramType :: Maybe (Colon, Expr v a)+  -- = spaces+  , _unsafeKeywordParamWhitespaceRight :: [Whitespace]+  , _unsafeKeywordParamExpr :: Expr v a+  }+  -- | @def foo(*xs):@+  | StarParam+  { _paramAnn :: a+  -- '*' spaces+  , _unsafeStarParamWhitespace :: [Whitespace]+  , _unsafeStarParamName :: Ident v a+  -- ':' spaces <expr>+  , _paramType :: Maybe (Colon, Expr v a)+  }+  -- | @def foo(*):@+  | UnnamedStarParam+  { _paramAnn :: a+  -- '*' spaces+  , _unsafeUnnamedStarParamWhitespace :: [Whitespace]+  }+  -- | @def foo(**dict):@+  | DoubleStarParam+  { _paramAnn :: a+  -- '**' spaces+  , _unsafeDoubleStarParamWhitespace :: [Whitespace]+  , _paramName :: Ident v a+  -- ':' spaces <expr>+  , _paramType :: Maybe (Colon, Expr v a)+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance IsString (Param '[] ()) where+  fromString a = PositionalParam () (fromString a) Nothing++instance HasTrailingWhitespace (Param v a) where+  trailingWhitespace =+    lens+      (\case+          PositionalParam _ a b ->+            maybe (a ^. trailingWhitespace) (^. _2.trailingWhitespace) b+          KeywordParam _ _ _ _ a -> a ^. trailingWhitespace+          UnnamedStarParam _ a -> a+          StarParam _ _ b c ->+            maybe+              (b ^. trailingWhitespace)+              (^. _2.trailingWhitespace)+              c+          DoubleStarParam _ _ a b ->+            maybe+              (a ^. trailingWhitespace)+              (^. _2.trailingWhitespace)+              b)+      (\p ws -> case p of+          PositionalParam a b c ->+            PositionalParam a+              (if isNothing c then b & trailingWhitespace .~ ws else b)+              (c & _Just._2.trailingWhitespace .~ ws)+          KeywordParam a b c d e ->+            KeywordParam a b c d $ e & trailingWhitespace .~ ws+          UnnamedStarParam a _ -> UnnamedStarParam a ws+          StarParam a b c d ->+            StarParam a+              b+              (if isNothing d then c & trailingWhitespace .~ ws else c)+              (d & _Just._2.trailingWhitespace .~ ws)+          DoubleStarParam a b c d ->+            DoubleStarParam a b+              (if isNothing d then c & trailingWhitespace .~ ws else c)+              (d & _Just._2.trailingWhitespace .~ ws))++-- | Lens on the syntrax tree annotation on a parameter+paramAnn :: Lens' (Param v a) a+paramAnn = lens _paramAnn (\s a -> s { _paramAnn = a})++-- | A faux-lens on the optional Python type annotation which may follow a parameter+--+-- This is not a lawful 'Lens' because setting an 'UnnamedStarParam''s type won't+-- have any effect.+--+-- This optic, like many others in hpython, loses validation information+-- (the @v@ type parameter)+--+-- The following is an example, where @int@ is the paramtype:+--+-- @+-- def foo(x: int):+-- @+paramType_+  :: Functor f+  => (Maybe (Colon, Expr v a) -> f (Maybe (Colon, Expr '[] a)))+  -> Param v a -> f (Param '[] a)+paramType_ =+  lens+    (\case+        UnnamedStarParam{} -> Nothing+        a -> _paramType a)+    (\s ty -> case s ^. unvalidated of+       PositionalParam a b _ -> PositionalParam a b ty+       KeywordParam a b _ c d -> KeywordParam a b ty c d+       StarParam a b c _ -> StarParam a b c ty+       UnnamedStarParam a b -> UnnamedStarParam a b+       DoubleStarParam a b c _ -> DoubleStarParam a b c ty)++-- | 'Traversal' targeting the Python type annotations which may follow a parameter+paramType :: Traversal (Param v a) (Param '[] a) (Colon, Expr v a) (Colon, Expr '[] a)+paramType = paramType_._Just++-- | (affine) 'Control.Lens.Traversal.Traversal' on the name of a parameter+--+-- The name is @x@ in the following examples:+--+-- @+-- def foo(x):+-- def foo(x=None):+-- def foo(*x):+-- def foo(**x):+-- @+--+-- But the following example does not have a 'paramName':+--+-- @+-- def foo(*):+-- @+paramName :: Traversal (Param v a) (Param '[] a) (Ident v a) (Ident '[] a)+paramName f (PositionalParam a b c) =+  PositionalParam a <$> f b <*> pure (over (mapped._2) (view unvalidated) c)+paramName f (KeywordParam a b c d e) =+  (\b' -> KeywordParam a b' (over (mapped._2) (view unvalidated) c) d (e ^. unvalidated)) <$>+  f b+paramName f (StarParam a b c d) =+  (\c' -> StarParam a b c' (over (mapped._2) (view unvalidated) d)) <$>+  f c+paramName _ (UnnamedStarParam a b) = pure $ UnnamedStarParam a b+paramName f (DoubleStarParam a b c d) =+  (\c' -> DoubleStarParam a b c' (over (mapped._2) (view unvalidated) d)) <$>+  f c++instance HasExprs Param where+  _Exprs f (KeywordParam a name ty ws2 expr) =+    KeywordParam a (coerce name) <$>+    traverseOf (traverse._2) f ty <*>+    pure ws2 <*>+    f expr+  _Exprs f (PositionalParam a b c) =+    PositionalParam a (coerce b) <$> traverseOf (traverse._2) f c+  _Exprs f (StarParam a b c d) =+    StarParam a b (coerce c) <$> traverseOf (traverse._2) f d+  _Exprs _ (UnnamedStarParam a b) = pure $ UnnamedStarParam a b+  _Exprs f (DoubleStarParam a b c d) =+    DoubleStarParam a b (coerce c) <$> traverseOf (traverse._2) f d++-- | Actual parameters for functions+--+-- In the following examples, @x@ is an actual parameter.+--+-- @+-- y = foo(x)+-- y = bar(quux=x)+-- y = baz(*x)+-- y = flux(**x)+-- @+data Arg (v :: [*]) a+  = PositionalArg+  { _argAnn :: a+  , _argExpr :: Expr v a+  }+  | KeywordArg+  { _argAnn :: a+  , _unsafeKeywordArgName :: Ident v a+  , _unsafeKeywordArgWhitespaceRight :: [Whitespace]+  , _argExpr :: Expr v a+  }+  | StarArg+  { _argAnn :: a+  , _unsafeStarArgWhitespace :: [Whitespace]+  , _argExpr :: Expr v a+  }+  | DoubleStarArg+  { _argAnn :: a+  , _unsafeDoubleStarArgWhitespace :: [Whitespace]+  , _argExpr :: Expr v a+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance IsString (Arg '[] ()) where; fromString = PositionalArg () . fromString++-- | Lens on the Python expression which is passed as the argument+argExpr :: Lens (Arg v a) (Arg '[] a) (Expr v a) (Expr '[] a)+argExpr = lens _argExpr (\s a -> (s ^. unvalidated) { _argExpr = a })++instance HasExprs Arg where+  _Exprs f (KeywordArg a name ws2 expr) = KeywordArg a (coerce name) ws2 <$> f expr+  _Exprs f (PositionalArg a expr) = PositionalArg a <$> f expr+  _Exprs f (StarArg a ws expr) = StarArg a ws <$> f expr+  _Exprs f (DoubleStarArg a ws expr) = StarArg a ws <$> f expr++-- | A Python for comprehension, such as+--+-- @+-- x for y in z+-- @+data Comprehension e (v :: [*]) a+  = Comprehension a (e v a) (CompFor v a) [Either (CompFor v a) (CompIf v a)] -- ^ <expr> <comp_for> (comp_for | comp_if)*+  deriving (Eq, Show)++instance HasTrailingWhitespace (Comprehension e v a) where+  trailingWhitespace =+    lens+      (\(Comprehension _ _ a b) ->+         case b of+           [] -> a ^. trailingWhitespace+           _ -> b ^?! _last.failing (_Left.trailingWhitespace) (_Right.trailingWhitespace))+      (\(Comprehension a b c d) ws ->+         case d of+           [] -> Comprehension a b (c & trailingWhitespace .~ ws) d+           _ ->+             Comprehension a b c+               (d &+                _last.failing (_Left.trailingWhitespace) (_Right.trailingWhitespace) .~ ws))++instance Functor (e v) => Functor (Comprehension e v) where+  fmap f (Comprehension a b c d) =+    Comprehension (f a) (fmap f b) (fmap f c) (fmap (bimap (fmap f) (fmap f)) d)++instance Foldable (e v) => Foldable (Comprehension e v) where+  foldMap f (Comprehension a b c d) =+    f a <> foldMap f b <> foldMap f c <> foldMap (bifoldMap (foldMap f) (foldMap f)) d++instance Traversable (e v) => Traversable (Comprehension e v) where+  traverse f (Comprehension a b c d) =+    Comprehension <$>+    f a <*>+    traverse f b <*>+    traverse f c <*>+    traverse (bitraverse (traverse f) (traverse f)) d++-- | A condition inside a comprehension, e.g. @[x for x in xs if even(x)]@+data CompIf (v :: [*]) a+  = CompIf a [Whitespace] (Expr v a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (CompIf v a) where+  trailingWhitespace =+    lens+      (\(CompIf _ _ a) -> a ^. trailingWhitespace)+      (\(CompIf a b c) ws -> CompIf a b $ c & trailingWhitespace .~ ws)++-- | A nested comprehesion, e.g. @[(x, y) for x in xs for y in ys]@+data CompFor (v :: [*]) a+  = CompFor a [Whitespace] (Expr v a) [Whitespace] (Expr v a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (CompFor v a) where+  trailingWhitespace =+    lens+      (\(CompFor _ _ _ _ a) -> a ^. trailingWhitespace)+      (\(CompFor a b c d e) ws -> CompFor a b c d $ e & trailingWhitespace .~ ws)++-- | @a : b@ or @**a@+--+-- Used to construct dictionaries, e.g. @{ 1: a, 2: b, **c }@+--+-- https://docs.python.org/3/reference/expressions.html#dictionary-displays+data DictItem (v :: [*]) a+  = DictItem+  { _dictItemAnn :: a+  , _unsafeDictItemKey :: Expr v a+  , _unsafeDictItemColon :: Colon+  , _unsafeDictItemValue :: Expr v a+  }+  | DictUnpack+  { _dictItemAnn :: a+  , _unsafeDictItemUnpackWhitespace :: [Whitespace]+  , _unsafeDictItemUnpackValue :: Expr v a+  } deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (DictItem v a) where+  trailingWhitespace =+    lens+      (\(DictItem _ _ _ a) -> a ^. trailingWhitespace)+      (\(DictItem a b c d) ws -> DictItem a b c (d & trailingWhitespace .~ ws))++-- | Syntax for things that can be used as subscripts (inside the square brackets)+--+-- e.g.+--+-- @a[b]@+--+-- @a[:]@+--+-- @a[b:]@+--+-- @a[:b]@+--+-- @a[b:c]@+--+-- @a[b:c:d]@+--+-- https://docs.python.org/3/reference/expressions.html#subscriptions+data Subscript (v :: [*]) a+  = SubscriptExpr (Expr v a)+  | SubscriptSlice+      -- [expr]+      (Maybe (Expr v a))+      -- ':' <spaces>+      Colon+      -- [expr]+      (Maybe (Expr v a))+      -- [':' [expr]]+      (Maybe (Colon, Maybe (Expr v a)))+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (Subscript v a) where+  trailingWhitespace =+    lens+      (\case+          SubscriptExpr e -> e ^. trailingWhitespace+          SubscriptSlice _ b c d ->+            case d of+              Nothing ->+                case c of+                  Nothing -> b ^. trailingWhitespace+                  Just e -> e ^. trailingWhitespace+              Just (e, f) ->+                case f of+                  Nothing -> e ^. trailingWhitespace+                  Just g -> g ^. trailingWhitespace)+      (\x ws ->+         case x of+          SubscriptExpr e -> SubscriptExpr $ e & trailingWhitespace .~ ws+          SubscriptSlice a b c d ->+            (\(b', c', d') -> SubscriptSlice a b' c' d') $+            case d of+              Nothing ->+                case c of+                  Nothing -> (MkColon ws, c, d)+                  Just e -> (b, Just $ e & trailingWhitespace .~ ws, d)+              Just (e, f) ->+                case f of+                  Nothing -> (b, c, Just (MkColon ws, f))+                  Just g -> (b, c, Just (e, Just $ g & trailingWhitespace .~ ws)))++-- | @a@ or @*a@+--+-- Used to construct lists, e.g. @[ 1, 'x', **c ]@+--+-- https://docs.python.org/3/reference/expressions.html#list-displays+data ListItem (v :: [*]) a+  = ListItem+  { _listItemAnn :: a+  , _unsafeListItemValue :: Expr v a+  }+  | ListUnpack+  { _listItemAnn :: a+  , _unsafeListUnpackParens :: [([Whitespace], [Whitespace])]+  , _unsafeListUnpackWhitespace :: [Whitespace]+  , _unsafeListUnpackValue :: Expr v a+  } deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasExprs ListItem where+  _Exprs f (ListItem a b) = ListItem a <$> f b+  _Exprs f (ListUnpack a b c d) = ListUnpack a b c <$> f d++instance HasTrailingWhitespace (ListItem v a) where+  trailingWhitespace =+    lens+      (\case+          ListItem _ a -> a ^. trailingWhitespace+          ListUnpack _ [] _ a -> a ^. trailingWhitespace+          ListUnpack _ ((_, ws) : _) _ _ -> ws)+      (\a ws ->+         case a of+           ListItem b c -> ListItem b $ c & trailingWhitespace .~ ws+           ListUnpack b [] d e -> ListUnpack b [] d $ e & trailingWhitespace .~ ws+           ListUnpack b ((c, _) : rest) e f -> ListUnpack b ((c, ws) : rest) e f)++-- | @a@ or @*a@+--+-- Used to construct sets, e.g. @{ 1, 'x', **c }@+--+-- https://docs.python.org/3/reference/expressions.html#set-displays+data SetItem (v :: [*]) a+  = SetItem+  { _setItemAnn :: a+  , _unsafeSetItemValue :: Expr v a+  }+  | SetUnpack+  { _setItemAnn :: a+  , _unsafeSetUnpackParens :: [([Whitespace], [Whitespace])]+  , _unsafeSetUnpackWhitespace :: [Whitespace]+  , _unsafeSetUnpackValue :: Expr v a+  } deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasExprs SetItem where+  _Exprs f (SetItem a b) = SetItem a <$> f b+  _Exprs f (SetUnpack a b c d) = SetUnpack a b c <$> f d++instance HasTrailingWhitespace (SetItem v a) where+  trailingWhitespace =+    lens+      (\case+          SetItem _ a -> a ^. trailingWhitespace+          SetUnpack _ [] _ a -> a ^. trailingWhitespace+          SetUnpack _ ((_, ws) : _) _ _ -> ws)+      (\a ws ->+         case a of+           SetItem b c -> SetItem b $ c & trailingWhitespace .~ ws+           SetUnpack b [] d e -> SetUnpack b [] d $ e & trailingWhitespace .~ ws+           SetUnpack b ((c, _) : rest) e f -> SetUnpack b ((c, ws) : rest) e f)++-- | @a@ or @*a@+--+-- Used to construct tuples, e.g. @(1, 'x', **c)@+data TupleItem (v :: [*]) a+  = TupleItem+  { _tupleItemAnn :: a+  , _unsafeTupleItemValue :: Expr v a+  }+  | TupleUnpack+  { _tupleItemAnn :: a+  , _unsafeTupleUnpackParens :: [([Whitespace], [Whitespace])]+  , _unsafeTupleUnpackWhitespace :: [Whitespace]+  , _unsafeTupleUnpackValue :: Expr v a+  } deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasExprs TupleItem where+  _Exprs f (TupleItem a b) = TupleItem a <$> f b+  _Exprs f (TupleUnpack a b c d) = TupleUnpack a b c <$> f d++instance HasTrailingWhitespace (TupleItem v a) where+  trailingWhitespace =+    lens+      (\case+          TupleItem _ a -> a ^. trailingWhitespace+          TupleUnpack _ [] _ a -> a ^. trailingWhitespace+          TupleUnpack _ ((_, ws) : _) _ _ -> ws)+      (\a ws ->+         case a of+           TupleItem b c -> TupleItem b $ c & trailingWhitespace .~ ws+           TupleUnpack b [] d e -> TupleUnpack b [] d $ e & trailingWhitespace .~ ws+           TupleUnpack b ((c, _) : rest) e f -> TupleUnpack b ((c, ws) : rest) e f)++-- | This large sum type covers all valid Python /expressions/+data Expr (v :: [*]) a+  -- | @()@+  --+  -- https://docs.python.org/3/reference/expressions.html#parenthesized-forms+  = Unit+  { _unsafeExprAnn :: a+  , _unsafeUnitWhitespaceInner :: [Whitespace]+  , _unsafeUnitWhitespaceRight :: [Whitespace]+  }+  -- | @lambda x, y: x@+  --+  -- https://docs.python.org/3/reference/expressions.html#lambda+  | Lambda+  { _unsafeExprAnn :: a+  , _unsafeLambdaWhitespace :: [Whitespace]+  , _unsafeLambdaArgs :: CommaSep (Param v a)+  , _unsafeLambdaColon :: Colon+  , _unsafeLambdaBody :: Expr v a+  }+  -- | @yield@+  --+  -- @yield a@+  --+  -- @yield a, b@+  --+  -- https://docs.python.org/3/reference/expressions.html#yield-expressions+  | Yield+  { _unsafeExprAnn :: a+  , _unsafeYieldWhitespace :: [Whitespace]+  , _unsafeYieldValue :: CommaSep (Expr v a)+  }+  -- | @yield from a@+  --+  -- https://docs.python.org/3/reference/expressions.html#yield-expressions+  | YieldFrom+  { _unsafeExprAnn :: a+  , _unsafeYieldWhitespace :: [Whitespace]+  , _unsafeFromWhitespace :: [Whitespace]+  , _unsafeYieldFromValue :: Expr v a+  }+  -- | @a if b else c@+  --+  -- https://docs.python.org/3/reference/expressions.html#conditional-expressions+  | Ternary+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeTernaryValue :: Expr v a+  -- 'if' spaces+  , _unsafeTernaryWhitespaceIf :: [Whitespace]+  -- expr+  , _unsafeTernaryCond :: Expr v a+  -- 'else' spaces+  , _unsafeTernaryWhitespaceElse :: [Whitespace]+  -- expr+  , _unsafeTernaryElse :: Expr v a+  }+  -- | @[a for b in c if d]@+  --+  -- https://docs.python.org/3/reference/expressions.html#list-displays+  | ListComp+  { _unsafeExprAnn :: a+  -- [ spaces+  , _unsafeListCompWhitespaceLeft :: [Whitespace]+  -- comprehension+  , _unsafeListCompValue :: Comprehension Expr v a+  -- ] spaces+  , _unsafeListCompWhitespaceRight :: [Whitespace]+  }+  -- | @[a, b, c]@+  --+  -- https://docs.python.org/3/reference/expressions.html#list-displays+  | List+  { _unsafeExprAnn :: a+  -- [ spaces+  , _unsafeListWhitespaceLeft :: [Whitespace]+  -- exprs+  , _unsafeListValues :: Maybe (CommaSep1' (ListItem v a))+  -- ] spaces+  , _unsafeListWhitespaceRight :: [Whitespace]+  }+  -- | @{a: b for c in d if e}@+  --+  -- https://docs.python.org/3/reference/expressions.html#dictionary-displays+  | DictComp+  { _unsafeExprAnn :: a+  -- { spaces+  , _unsafeDictCompWhitespaceLeft :: [Whitespace]+  -- comprehension+  , _unsafeDictCompValue :: Comprehension DictItem v a+  -- } spaces+  , _unsafeDictCompWhitespaceRight :: [Whitespace]+  }+  -- | @{}@+  --+  -- @{a: 1, b: 2, c: 3}@+  --+  -- https://docs.python.org/3/reference/expressions.html#dictionary-displays+  | Dict+  { _unsafeExprAnn :: a+  , _unsafeDictWhitespaceLeft :: [Whitespace]+  , _unsafeDictValues :: Maybe (CommaSep1' (DictItem v a))+  , _unsafeDictWhitespaceRight :: [Whitespace]+  }+  -- | @{a for b in c if d}@+  --+  -- https://docs.python.org/3/reference/expressions.html#set-displays+  | SetComp+  { _unsafeExprAnn :: a+  -- { spaces+  , _unsafeSetCompWhitespaceLeft :: [Whitespace]+  -- comprehension+  , _unsafeSetCompValue :: Comprehension SetItem v a+  -- } spaces+  , _unsafeSetCompWhitespaceRight :: [Whitespace]+  }+  -- | @{a, b, c}@+  --+  -- https://docs.python.org/3/reference/expressions.html#set-displays+  | Set+  { _unsafeExprAnn :: a+  , _unsafeSetWhitespaceLeft :: [Whitespace]+  , _unsafeSetValues :: CommaSep1' (SetItem v a)+  , _unsafeSetWhitespaceRight :: [Whitespace]+  }+  -- | @a.b@+  --+  -- https://docs.python.org/3/reference/expressions.html#attribute-references+  | Deref+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeDerefValueLeft :: Expr v a+  -- . spaces+  , _unsafeDerefWhitespaceLeft :: [Whitespace]+  -- ident+  , _unsafeDerefValueRight :: Ident v a+  }+  -- | @a[b]@+  --+  -- @a[:]@+  --+  -- @a[:, b:]@+  --+  -- etc.+  --+  -- https://docs.python.org/3/reference/expressions.html#subscriptions+  | Subscript+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeSubscriptValueLeft :: Expr v a+  -- [ spaces+  , _unsafeSubscriptWhitespaceLeft :: [Whitespace]+  -- expr+  , _unsafeSubscriptValueRight :: CommaSep1' (Subscript v a)+  -- ] spaces+  , _unsafeSubscriptWhitespaceRight :: [Whitespace]+  }+  -- | @f(x)@+  --+  -- https://docs.python.org/3/reference/expressions.html#calls+  | Call+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeCallFunction :: Expr v a+  -- ( spaces+  , _unsafeCallWhitespaceLeft :: [Whitespace]+  -- exprs+  , _unsafeCallArguments :: Maybe (CommaSep1' (Arg v a))+  -- ) spaces+  , _unsafeCallWhitespaceRight :: [Whitespace]+  }+  -- | @None@+  --+  -- https://docs.python.org/3/library/constants.html#None+  | None+  { _unsafeExprAnn :: a+  , _unsafeNoneWhitespace :: [Whitespace]+  }+  -- | @...@+  --+  -- https://docs.python.org/3/library/constants.html#Ellipsis+  | Ellipsis+  { _unsafeExprAnn :: a+  , _unsafeEllipsisWhitespace :: [Whitespace]+  }+  -- | @a + b@+  --+  -- https://docs.python.org/3/reference/expressions.html#the-power-operator+  --+  -- https://docs.python.org/3/reference/expressions.html#binary-arithmetic-operations+  --+  -- https://docs.python.org/3/reference/expressions.html#shifting-operations+  --+  -- https://docs.python.org/3/reference/expressions.html#binary-bitwise-operations+  --+  -- https://docs.python.org/3/reference/expressions.html#comparisons+  --+  -- https://docs.python.org/3/reference/expressions.html#membership-test-operations+  --+  -- https://docs.python.org/3/reference/expressions.html#is-not+  --+  -- https://docs.python.org/3/reference/expressions.html#boolean-operations+  | BinOp+  { _unsafeExprAnn :: a+  , _unsafeBinOpExprLeft :: Expr v a+  , _unsafeBinOpOp :: BinOp a+  , _unsafeBinOpExprRight :: Expr v a+  }+  -- | @-a@+  --+  -- @~a@+  --+  -- @+a@+  --+  -- https://docs.python.org/3/reference/expressions.html#unary-arithmetic-and-bitwise-operations+  | UnOp+  { _unsafeExprAnn :: a+  , _unsafeUnOpOp :: UnOp a+  , _unsafeUnOpValue :: Expr v a+  }+  | Parens+  { _unsafeExprAnn :: a+  -- ( spaces+  , _unsafeParensWhitespaceLeft :: [Whitespace]+  -- expr+  , _unsafeParensValue :: Expr v a+  -- ) spaces+  , _unsafeParensWhitespaceAfter :: [Whitespace]+  }+  -- | @a@+  --+  -- https://docs.python.org/3/reference/expressions.html#atom-identifiers+  | Ident+  { _unsafeIdentValue :: Ident v a+  }+  -- | @1@+  --+  -- @0xF3A+  --+  -- @0o177+  --+  -- @0b1011@+  --+  -- https://docs.python.org/3/reference/lexical_analysis.html#grammar-token-integer+  | Int+  { _unsafeExprAnn :: a+  , _unsafeIntValue :: IntLiteral a+  , _unsafeIntWhitespace :: [Whitespace]+  }+  -- | @1.@+  --+  -- @3.14@+  --+  -- @10e100@+  --+  -- https://docs.python.org/3/reference/lexical_analysis.html#floating-point-literals+  | Float+  { _unsafeExprAnn :: a+  , _unsafeFloatValue :: FloatLiteral a+  , _unsafeFloatWhitespace :: [Whitespace]+  }+  -- | @10j@+  --+  -- @5.j@+  --+  -- https://docs.python.org/3/reference/lexical_analysis.html#floating-point-literals+  | Imag+  { _unsafeExprAnn :: a+  , _unsafeImagValue :: ImagLiteral a+  , _unsafeImagWhitespace :: [Whitespace]+  }+  -- | @True@+  --+  -- @False@+  --+  -- https://docs.python.org/3/library/constants.html#True+  --+  -- https://docs.python.org/3/library/constants.html#False+  | Bool+  { _unsafeExprAnn :: a+  , _unsafeBoolValue :: Bool+  , _unsafeBoolWhitespace :: [Whitespace]+  }+  -- | @\"asdf\"@+  --+  -- @b\"asdf\"@+  --+  -- @\"asdf\" \'asdf\'@+  --+  -- @\'\'\'asdf\'\'\'@+  --+  -- https://docs.python.org/3/reference/lexical_analysis.html#grammar-token-stringliteral+  | String+  { _unsafeExprAnn :: a+  , _unsafeStringValue :: NonEmpty (StringLiteral a)+  }+  -- | @a, b, c@+  --+  -- @(a, b)@+  --+  -- @(a,)@+  --+  -- https://docs.python.org/3/reference/expressions.html#expression-lists+  | Tuple+  { _unsafeExprAnn :: a+  -- expr+  , _unsafeTupleHead :: TupleItem v a+  -- , spaces+  , _unsafeTupleWhitespace :: Comma+  -- [exprs]+  , _unsafeTupleTail :: Maybe (CommaSep1' (TupleItem v a))+  }+  -- | @not a@+  --+  -- https://docs.python.org/3/reference/expressions.html#boolean-operations+  | Not+  { _unsafeExprAnn :: a+  , _unsafeNotWhitespace :: [Whitespace]+  , _unsafeNotValue :: Expr v a+  }+  -- | @(a for b in c)@+  --+  -- https://docs.python.org/3/reference/expressions.html#generator-expressions+  | Generator+  { _unsafeExprAnn :: a+  , _generatorValue :: Comprehension Expr v a+  }+  -- | @await a@+  --+  -- https://docs.python.org/3/reference/expressions.html#await+  | Await+  { _unsafeExprAnn :: a+  , _unsafeAwaitWhitespace :: [Whitespace]+  , _unsafeAwaitValue :: Expr v a+  }+  deriving (Eq, Show, Functor, Foldable, Traversable, Generic)++-- | Lens on the top-level annotation in an expression+exprAnn :: Lens' (Expr v a) a+exprAnn =+  lens+    (\case+        Unit a _ _ -> a+        Lambda a _ _ _ _ -> a+        Yield a _ _ -> a+        YieldFrom a _ _ _ -> a+        Ternary a _ _ _ _ _ -> a+        None a _ -> a+        Ellipsis a _ -> a+        List a _ _ _ -> a+        ListComp a _ _ _ -> a+        Deref a _ _ _ -> a+        Subscript a _ _ _ _ -> a+        Call a _ _ _ _ -> a+        BinOp a _ _ _ -> a+        UnOp a _ _ -> a+        Parens a _ _ _ -> a+        Ident a -> a ^. identAnn+        Int a _ _ -> a+        Float a _ _ -> a+        Imag a _ _ -> a+        Bool a _ _ -> a+        String a _ -> a+        Not a _ _ -> a+        Tuple a _ _ _ -> a+        DictComp a _ _ _ -> a+        Dict a _ _ _ -> a+        SetComp a _ _ _ -> a+        Set a _ _ _ -> a+        Generator a _ -> a+        Await a _ _ -> a)+    (\e ann ->+      case e of+        Unit _ a b -> Unit ann a b+        Lambda _ a b c d -> Lambda ann a b c d+        Yield _ a b -> Yield ann a b+        YieldFrom ann a b c -> YieldFrom ann a b c+        Ternary ann a b c d e -> Ternary ann a b c d e+        None _ a -> None ann a+        Ellipsis _ a -> Ellipsis ann a+        List _ a b c -> List ann a b c+        ListComp _ a b c -> ListComp ann a b c+        Deref _ a b c -> Deref ann a b c+        Subscript _ a b c d -> Subscript ann a b c d+        Call _ a b c d -> Call ann a b c d+        BinOp _ a b c -> BinOp ann a b c+        UnOp _ a b -> UnOp ann a b+        Parens _ a b c -> Parens ann a b c+        Ident a -> Ident $ a & identAnn .~ ann+        Int _ a b -> Int ann a b+        Float _ a b -> Float ann a b+        Imag _ a b -> Imag ann a b+        Bool _ a b -> Bool ann a b+        String _ a -> String ann a+        Not _ a b -> Not ann a b+        Tuple _ a b c -> Tuple ann a b c+        DictComp _ a b c -> DictComp ann a b c+        Dict _ a b c -> Dict ann a b c+        SetComp _ a b c -> SetComp ann a b c+        Set _ a b c -> Set ann a b c+        Generator _ a -> Generator ann a+        Await _ a b -> Not ann a b)++instance HasTrailingWhitespace (Expr v a) where+  trailingWhitespace =+    lens+      (\case+          Unit _ _ a -> a+          Lambda _ _ _ _ a -> a ^. trailingWhitespace+          Yield _ ws CommaSepNone -> ws+          Yield _ _ e -> e ^?! csTrailingWhitespace+          YieldFrom _ _ _ e -> e ^. trailingWhitespace+          Ternary _ _ _ _ _ e -> e ^. trailingWhitespace+          None _ ws -> ws+          Ellipsis _ ws -> ws+          List _ _ _ ws -> ws+          ListComp _ _ _ ws -> ws+          Deref _ _ _ a -> a ^. trailingWhitespace+          Subscript _ _ _ _ ws -> ws+          Call _ _ _ _ ws -> ws+          BinOp _ _ _ e -> e ^. trailingWhitespace+          UnOp _ _ e -> e ^. trailingWhitespace+          Parens _ _ _ ws -> ws+          Ident a -> a ^. getting trailingWhitespace+          Int _ _ ws -> ws+          Float _ _ ws -> ws+          Imag _ _ ws -> ws+          Bool _ _ ws -> ws+          String _ v -> v ^. trailingWhitespace+          Not _ _ e -> e ^. trailingWhitespace+          Tuple _ _ (MkComma ws) Nothing -> ws+          Tuple _ _ _ (Just cs) -> cs ^. trailingWhitespace+          DictComp _ _ _ ws -> ws+          Dict _ _ _ ws -> ws+          SetComp _ _ _ ws -> ws+          Set _ _ _ ws -> ws+          Generator  _ a -> a ^. trailingWhitespace+          Await _ _ e -> e ^. trailingWhitespace)+      (\e ws ->+        case e of+          Unit a b _ -> Unit a b ws+          Lambda a b c d f -> Lambda a b c d (f & trailingWhitespace .~ ws)+          Yield a _ CommaSepNone -> Yield a ws CommaSepNone+          Yield a b c -> Yield a b (c & csTrailingWhitespace .~ ws)+          YieldFrom a b c d -> YieldFrom a b c (d & trailingWhitespace .~ ws)+          Ternary a b c d e f -> Ternary a b c d e (f & trailingWhitespace .~ ws)+          None a _ -> None a ws+          Ellipsis a _ -> Ellipsis a ws+          List a b c _ -> List a b (coerce c) ws+          ListComp a b c _ -> ListComp a b (coerce c) ws+          Deref a b c d -> Deref a (coerce b) c (d & trailingWhitespace .~ ws)+          Subscript a b c d _ -> Subscript a (coerce b) c d ws+          Call a b c d _ -> Call a (coerce b) c (coerce d) ws+          BinOp a b c e -> BinOp a (coerce b) c (e & trailingWhitespace .~ ws)+          UnOp a b c -> UnOp a b (c & trailingWhitespace .~ ws)+          Parens a b c _ -> Parens a b (coerce c) ws+          Ident a -> Ident $ a & trailingWhitespace .~ ws+          Int a b _ -> Int a b ws+          Float a b _ -> Float a b ws+          Imag a b _ -> Imag a b ws+          Bool a b _ -> Bool a b ws+          String a v -> String a (v & trailingWhitespace .~ ws)+          Not a b c -> Not a b (c & trailingWhitespace .~ ws)+          Tuple a b _ Nothing -> Tuple a (coerce b) (MkComma ws) Nothing+          Tuple a b c (Just cs) ->+            Tuple a (coerce b) c (Just $ cs & trailingWhitespace .~ ws)+          DictComp a b c _ -> DictComp a b c ws+          Dict a b c _ -> Dict a b c ws+          SetComp a b c _ -> SetComp a b c ws+          Set a b c _ -> Set a b c ws+          Generator a b -> Generator a $ b & trailingWhitespace .~ ws+          Await a b c -> Await a b (c & trailingWhitespace .~ ws))++instance IsString (Expr '[] ()) where+  fromString s = Ident $ MkIdent () s []++instance Num (Expr '[] ()) where+  fromInteger n+    | n >= 0 = Int () (IntLiteralDec () $ integralDecDigits n ^?! _Right) []+    | otherwise =+        UnOp+          ()+          (Negate () [])+          (Int () (IntLiteralDec () $ integralDecDigits (-n) ^?! _Right) [])++  negate = UnOp () (Negate () [])++  (+) a = BinOp () (a & trailingWhitespace .~ [Space]) (Plus () [Space])+  (*) a = BinOp () (a & trailingWhitespace .~ [Space]) (Multiply () [Space])+  (-) a = BinOp () (a & trailingWhitespace .~ [Space]) (Minus () [Space])+  signum = undefined+  abs = undefined++instance Plated (Expr '[] a) where; plate = gplate++instance HasExprs Expr where+  _Exprs = id++-- |+-- @shouldGroupLeft op left@ returns true if @left@ needs to be parenthesised+-- when it is the left argument of @op@+shouldGroupLeft :: BinOp a -> Expr v a -> Bool+shouldGroupLeft op left =+  let+    entry = lookupOpEntry op operatorTable++    lEntry =+      case left of+        BinOp _ _ lOp _ -> Just $ lookupOpEntry lOp operatorTable+        _ -> Nothing++    leftf =+      case entry ^. opAssoc of+        R | Just (OpEntry _ prec R) <- lEntry -> prec <= entry ^. opPrec+        _ -> False++    leftf' =+      case (left, op) of+        (UnOp{}, Exp{}) -> True+        (Tuple{}, _) -> True+        (Not{}, BoolAnd{}) -> False+        (Not{}, BoolOr{}) -> False+        (Not{}, _) -> True+        _ -> maybe False (\p -> p < entry ^. opPrec) (lEntry ^? _Just.opPrec)+  in+    leftf || leftf'++-- |+-- @shouldGroupRight op right@ returns true if @right@ needs to be parenthesised+-- when it is the right argument of @op@+shouldGroupRight :: BinOp a -> Expr v a -> Bool+shouldGroupRight op right =+  let+    entry = lookupOpEntry op operatorTable++    rEntry =+      case right of+        BinOp _ _ rOp _ -> Just $ lookupOpEntry rOp operatorTable+        _ -> Nothing++    rightf =+      case entry ^. opAssoc of+        L | Just (OpEntry _ prec L) <- rEntry -> prec <= entry ^. opPrec+        _ -> False++    rightf' =+      case (op, right) of+        (_, Tuple{}) -> True+        (BoolAnd{}, Not{}) -> False+        (BoolOr{}, Not{}) -> False+        (_, Not{}) -> True+        _ -> maybe False (\p -> p < entry ^. opPrec) (rEntry ^? _Just.opPrec)+  in+    rightf || rightf'
+ src/Language/Python/Syntax/Ident.hs view
@@ -0,0 +1,76 @@+{-# language DataKinds, KindSignatures #-}+{-# language FlexibleInstances #-}+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}++{-|+Module      : Language.Python.Syntax.Ident+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.Ident+  ( Ident(..)+    -- * Lenses+  , identAnn+  , identValue+  , identWhitespace+    -- * Extra functions+  , isIdentifierStart+  , isIdentifierChar+  )+where++import Control.Lens.Lens (Lens, lens)+import Data.Char (isDigit, isLetter)+import Data.String (IsString(..))++import Language.Python.Optics.Validated (Validated)+import Language.Python.Syntax.Raw+import Language.Python.Syntax.Whitespace++-- | An identifier. Like many types in hpython, it has an optional annotation+-- and tracks its trailing whitespace.+--+-- 'Raw' 'Ident's have an 'IsString' instance.+--+-- See <https://docs.python.org/3.5/reference/lexical_analysis.html#identifiers>+data Ident (v :: [*]) a+  = MkIdent+  { _identAnn :: a+  , _identValue :: String+  , _identWhitespace :: [Whitespace]+  } deriving (Eq, Show, Functor, Foldable, Traversable)++-- | Determine whether this character could start a valid identifier+isIdentifierStart :: Char -> Bool+isIdentifierStart = do+  a <- isLetter+  b <- (=='_')+  pure $ a || b++-- | Determine whether this character could be part of a valid identifier+isIdentifierChar :: Char -> Bool+isIdentifierChar = do+  a <- isIdentifierStart+  b <- isDigit+  pure $ a || b++instance IsString (Raw Ident) where+  fromString s = MkIdent () s []++identValue :: Lens (Ident v a) (Ident '[] a) String String+identValue = lens _identValue (\s a -> s { _identValue = a })++identAnn :: Lens (Ident v a) (Ident v a) a a+identAnn = lens _identAnn (\s a -> s { _identAnn = a })++identWhitespace :: Lens (Ident v a) (Ident v a) [Whitespace] [Whitespace]+identWhitespace = lens _identWhitespace (\s ws -> s { _identWhitespace = ws })++instance HasTrailingWhitespace (Ident v a) where+  trailingWhitespace = identWhitespace++instance Validated Ident where
+ src/Language/Python/Syntax/Import.hs view
@@ -0,0 +1,113 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language DataKinds #-}+{-# language LambdaCase #-}++{-|+Module      : Language.Python.Syntax.Import+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Syntax used in import statements++https://docs.python.org/3.5/reference/simple_stmts.html#the-import-statement+-}++module Language.Python.Syntax.Import+  ( ImportAs(..)+  , ImportTargets(..)+    -- * Lenses+  , importAsAnn+  , importAsName+  , importAsQual+  )+where++import Control.Lens.Getter ((^.), getting)+import Control.Lens.Lens (Lens, Lens', lens)+import Control.Lens.Prism (_Just)+import Control.Lens.Setter ((.~))+import Control.Lens.Tuple (_2)+import Data.Function ((&))+import Data.List.NonEmpty (NonEmpty)++import Language.Python.Optics.Validated+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Whitespace++-- | Some data optionally followed by @as <ident>@+--+-- Used in:+--+-- @import a as b@+--+-- @from a import b as c, d as e@+--+-- @from a import (b as c, d as e)@+data ImportAs e v a+  = ImportAs+  { _importAsAnn :: a+  , _importAsName :: e a+  , _importAsQual :: Maybe (NonEmpty Whitespace, Ident v a)+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance Validated (ImportAs e)++importAsAnn :: Lens' (ImportAs e v a) a+importAsAnn = lens _importAsAnn (\s a -> s { _importAsAnn = a })++importAsName :: Lens (ImportAs e v a) (ImportAs e' '[] a) (e a) (e' a)+importAsName = lens _importAsName (\s a -> (s ^. unvalidated) { _importAsName = a })++importAsQual+  :: Lens+       (ImportAs e v a)+       (ImportAs e '[] a)+       (Maybe (NonEmpty Whitespace, Ident v a))+       (Maybe (NonEmpty Whitespace, Ident '[] a))+importAsQual = lens _importAsQual (\s a -> (s ^. unvalidated) { _importAsQual = a })++instance HasTrailingWhitespace (e a) => HasTrailingWhitespace (ImportAs e v a) where+  trailingWhitespace =+    lens+      (\(ImportAs _ a b) ->+         maybe (a ^. getting trailingWhitespace) (^. _2.trailingWhitespace) b)+      (\(ImportAs x a b) ws ->+         ImportAs+           x+           (maybe (a & trailingWhitespace .~ ws) (const a) b)+           (b & _Just._2.trailingWhitespace .~ ws))++-- | The targets of a @from ... import ...@ statement+data ImportTargets v a+  -- | @from x import *@+  = ImportAll a [Whitespace]+  -- | @from x import a, b, c@+  | ImportSome a (CommaSep1 (ImportAs (Ident v) v a))+  -- | @from x import (a, b, c)@+  | ImportSomeParens+      a+      -- ( spaces+      [Whitespace]+      -- imports as+      (CommaSep1' (ImportAs (Ident v) v a))+      -- ) spaces+      [Whitespace]+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (ImportTargets v a) where+  trailingWhitespace =+    lens+      (\case+          ImportAll _ ws -> ws+          ImportSome _ cs -> cs ^. trailingWhitespace+          ImportSomeParens _ _ _ ws -> ws)+      (\ts ws ->+         case ts of+           ImportAll a _ -> ImportAll a ws+           ImportSome a cs -> ImportSome a (cs & trailingWhitespace .~ ws)+           ImportSomeParens x a b _ -> ImportSomeParens x a b ws)
+ src/Language/Python/Syntax/Module.hs view
@@ -0,0 +1,35 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}++{-|+Module      : Language.Python.Syntax.Module+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.Module+  ( Module (ModuleEmpty, ModuleBlankFinal, ModuleBlank, ModuleStatement)+  )+where++import Language.Python.Syntax.Statement+import Language.Python.Syntax.Whitespace++-- | A Python 'Module', which is stored as a sequence of statements.+-- A module corresponds to one source file of Python code.+data Module v a+  = ModuleEmpty+  | ModuleBlankFinal (Blank a)+  | ModuleBlank (Blank a) Newline (Module v a)+  | ModuleStatement (Statement v a) (Module v a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasStatements Module where+  _Statements f = go+    where+      go ModuleEmpty = pure ModuleEmpty+      go (ModuleBlankFinal a) = pure $ ModuleBlankFinal a+      go (ModuleBlank a b c) = ModuleBlank a b <$> go c+      go (ModuleStatement a b) = ModuleStatement <$> f a <*> go b
+ src/Language/Python/Syntax/ModuleNames.hs view
@@ -0,0 +1,97 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language DataKinds, FlexibleInstances, MultiParamTypeClasses #-}+{-# language LambdaCase #-}++{-|+Module      : Language.Python.Syntax.ModuleNames+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Module names, including those qualified by packages.++See <https://docs.python.org/3.5/tutorial/modules.html#packages>+-}++module Language.Python.Syntax.ModuleNames+  ( ModuleName (..)+  , RelativeModuleName (..)+  , makeModuleName+  , _moduleNameAnn+  )+where++import Control.Lens.Cons (_last)+import Control.Lens.Fold ((^?!))+import Control.Lens.Getter ((^.))+import Control.Lens.Lens (lens)+import Control.Lens.Setter ((.~))+import Data.Coerce (coerce)+import Data.Function ((&))+import Data.List.NonEmpty (NonEmpty(..))+import qualified Data.List.NonEmpty as NonEmpty++import Language.Python.Syntax.Ident+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Whitespace++-- | @.a.b@+--+-- @.@+--+-- @...@+--+--See <https://docs.python.org/3.5/tutorial/modules.html#intra-package-references>+data RelativeModuleName v a+  = RelativeWithName [Dot] (ModuleName v a)+  | Relative (NonEmpty Dot)+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (RelativeModuleName v a) where+  trailingWhitespace =+    lens+      (\case+          RelativeWithName _ mn -> mn ^. trailingWhitespace+          Relative (a :| as) -> (a : as) ^?! _last.trailingWhitespace)+      (\a ws -> case a of+          RelativeWithName x mn -> RelativeWithName x (mn & trailingWhitespace .~ ws)+          Relative (a :| as) ->+            Relative .+            NonEmpty.fromList $+            (a : as) & _last.trailingWhitespace .~ ws)++-- | A module name. It can be a single segment, or a sequence of them which+-- are implicitly separated by period character.+--+-- @a@+--+-- @a.b@+data ModuleName v a+  = ModuleNameOne a (Ident v a)+  | ModuleNameMany a (Ident v a) Dot (ModuleName v a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | Get the annotation from a 'ModuleName'+_moduleNameAnn :: ModuleName v a -> a+_moduleNameAnn (ModuleNameOne a _) = a+_moduleNameAnn (ModuleNameMany a _ _ _) = a++-- | Convenience constructor for 'ModuleName'+makeModuleName :: Ident v a -> [([Whitespace], Ident v a)] -> ModuleName v a+makeModuleName i [] = ModuleNameOne (_identAnn i) i+makeModuleName i ((a, b) : as) =+  ModuleNameMany (_identAnn i) i (MkDot a) $+  makeModuleName b as++instance HasTrailingWhitespace (ModuleName v a) where+  trailingWhitespace =+    lens+      (\case+          ModuleNameOne _ i -> i ^. trailingWhitespace+          ModuleNameMany _ _ _ mn -> mn ^. trailingWhitespace)+      (\mn ws -> case mn of+          ModuleNameOne a b -> ModuleNameOne a (b & trailingWhitespace .~ ws)+          ModuleNameMany a b d mn ->+            ModuleNameMany a (coerce b) d (mn & trailingWhitespace .~ ws))
+ src/Language/Python/Syntax/Numbers.hs view
@@ -0,0 +1,224 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language LambdaCase #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Syntax.Numbers+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Numerical literal values in Python+-}++module Language.Python.Syntax.Numbers+  ( -- * Datatypes+    IntLiteral(..)+  , Sign(..)+  , E(..)+  , FloatExponent(..)+  , FloatLiteral(..)+  , ImagLiteral(..)+    -- * Rendering+    -- | The output of these functions is guaranteed to be valid Python code+  , showIntLiteral+  , showFloatLiteral+  , showFloatExponent+  , showImagLiteral+  )+where++import Control.Lens.Review ((#))+import Data.Deriving (deriveEq1, deriveOrd1)+import Data.Digit.Binary (BinDigit)+import Data.Digit.Char (charHeXaDeCiMaL, charOctal, charBinary, charDecimal)+import Data.Digit.Octal (OctDigit)+import Data.Digit.Decimal (DecDigit)+import Data.Digit.Hexadecimal.MixedCase (HeXDigit)+import Data.List.NonEmpty (NonEmpty)+import Data.Semigroup ((<>))+import Data.Text (Text)+import Data.These (These(..))++import qualified Data.List.NonEmpty as NonEmpty+import qualified Data.Text as Text++-- | An integer literal value.+--+-- @5@ is an integer literal.+--+-- @6.2@ is a literal but is not an integer+--+-- @x@ might be an integer, but is not a literal+--+-- See <https://docs.python.org/3.5/reference/lexical_analysis.html#integer-literals>+data IntLiteral a+  -- | Decimal+  --+  -- @1234@+  = IntLiteralDec+  { _intLiteralAnn :: a+  , _unsafeIntLiteralDecValue :: NonEmpty DecDigit+  }+  -- | Binary+  --+  -- @0b10110@+  | IntLiteralBin+  { _intLiteralAnn :: a+  , _unsafeIntLiteralBinUppercase :: Bool+  , _unsafeIntLiteralBinValue :: NonEmpty BinDigit+  }+  -- | Octal+  --+  -- @0o1367@+  | IntLiteralOct+  { _intLiteralAnn :: a+  , _unsafeIntLiteralOctUppercase :: Bool+  , _unsafeIntLiteralOctValue :: NonEmpty OctDigit+  }+  -- | Mixed-case hexadecimal+  --+  -- @0x18B4f@+  | IntLiteralHex+  { _intLiteralAnn :: a+  , _unsafeIntLiteralHexUppercase :: Bool+  , _unsafeIntLiteralHexValue :: NonEmpty HeXDigit+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)+deriveEq1 ''IntLiteral+deriveOrd1 ''IntLiteral++-- | Positive or negative, as in @-7@+data Sign = Pos | Neg deriving (Eq, Ord, Show)++-- | When a floating point literal is in scientific notation, it includes the character+-- @e@, which can be lower or upper case.+data E = Ee | EE deriving (Eq, Ord, Show)++-- | The exponent of a floating point literal.+--+-- An @e@, followed by an optional 'Sign', followed by at least one digit.+data FloatExponent = FloatExponent E (Maybe Sign) (NonEmpty DecDigit)+  deriving (Eq, Ord, Show)++-- | A literal floating point value.+--+-- Eg. @7.63@+--+-- See <https://docs.python.org/3.5/reference/lexical_analysis.html#floating-point-literals>+data FloatLiteral a+  -- | \'Complete\' floats+  --+  -- @12.@+  --+  -- @12.34@+  --+  -- @12.e34@+  --+  -- @12.34e56@+  = FloatLiteralFull+  { _floatLiteralAnn :: a+  , _floatLiteralFullLeft :: NonEmpty DecDigit+  , _floatLiteralFullRight+      :: Maybe (These (NonEmpty DecDigit) FloatExponent)+  }+  -- | Floats that begin with a decimal point+  --+  -- @.12@+  --+  -- @.12e34@+  | FloatLiteralPoint+  { _floatLiteralAnn :: a+  -- . [0-9]++  , _floatLiteralPointRight :: NonEmpty DecDigit+  -- [ 'e' ['-' | '+'] [0-9]+ ]+  , _floatLiteralPointExponent :: Maybe FloatExponent+  }+  -- | Floats with no decimal points+  --+  -- @12e34@+  | FloatLiteralWhole+  { _floatLiteralAnn :: a+  -- [0-9]++  , _floatLiteralWholeRight :: NonEmpty DecDigit+  -- [ 'e' ['-' | '+'] [0-9]+ ]+  , _floatLiteralWholeExponent :: FloatExponent+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)+deriveEq1 ''FloatLiteral+deriveOrd1 ''FloatLiteral++-- | Imaginary number literals+--+-- See <https://docs.python.org/3.5/reference/lexical_analysis.html#imaginary-literals>+data ImagLiteral a+  -- | A decimal integer followed by a \'j\'+  --+  -- @12j@+  = ImagLiteralInt+  { _imagLiteralAnn :: a+  , _unsafeImagLiteralIntValue :: NonEmpty DecDigit+  , _imagLiteralUppercase :: Bool+  }+  -- | A float followed by a \'j\'+  --+  -- @12.j@+  --+  -- @12.3j@+  --+  -- @.3j@+  | ImagLiteralFloat+  { _imagLiteralAnn :: a+  , _unsafeImagLiteralFloatValue :: FloatLiteral a+  , _imagLiteralUppercase :: Bool+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)+deriveEq1 ''ImagLiteral+deriveOrd1 ''ImagLiteral++showIntLiteral :: IntLiteral a -> Text+showIntLiteral (IntLiteralDec _ n) =+  Text.pack $+  (charDecimal #) <$> NonEmpty.toList n+showIntLiteral (IntLiteralBin _ b n) =+  Text.pack $+  '0' : (if b then 'B' else 'b') : fmap (charBinary #) (NonEmpty.toList n)+showIntLiteral (IntLiteralOct _ b n) =+  Text.pack $+  '0' : (if b then 'O' else 'o') : fmap (charOctal #) (NonEmpty.toList n)+showIntLiteral (IntLiteralHex _ b n) =+  Text.pack $+  '0' : (if b then 'X' else 'x') : fmap (charHeXaDeCiMaL #) (NonEmpty.toList n)++showFloatExponent :: FloatExponent -> Text+showFloatExponent (FloatExponent e s ds) =+  Text.pack $+  (case e of; EE -> 'E'; Ee -> 'e') :+  foldMap (\case; Pos -> "+"; Neg -> "-") s <>+  fmap (charDecimal #) (NonEmpty.toList ds)++showFloatLiteral :: FloatLiteral a -> Text+showFloatLiteral (FloatLiteralFull _ a b) =+  Text.pack (fmap (charDecimal #) (NonEmpty.toList a) <> ".") <>+  foldMap+    (\case+       This x -> Text.pack $ fmap (charDecimal #) (NonEmpty.toList x)+       That x -> showFloatExponent x+       These x y ->+         Text.pack (fmap (charDecimal #) (NonEmpty.toList x)) <>+         showFloatExponent y)+    b+showFloatLiteral (FloatLiteralPoint _ a b) =+  Text.pack ('.' : fmap (charDecimal #) (NonEmpty.toList a)) <>+  foldMap showFloatExponent b+showFloatLiteral (FloatLiteralWhole _ a b) =+  Text.pack (fmap (charDecimal #) (NonEmpty.toList a)) <>+  showFloatExponent b++showImagLiteral :: ImagLiteral a -> Text+showImagLiteral (ImagLiteralInt _ ds b) =+  Text.pack $ fmap (charDecimal #) (NonEmpty.toList ds) ++ [if b then 'J' else 'j']+showImagLiteral (ImagLiteralFloat _ f b) =+  showFloatLiteral f <> Text.singleton (if b then 'J' else 'j')
+ src/Language/Python/Syntax/Operator/Binary.hs view
@@ -0,0 +1,254 @@+{-# language LambdaCase #-}+{-# language MultiParamTypeClasses, FlexibleInstances #-}+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Syntax.Operator.Binary+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++This module contains a datatype for binary operators and a precedence table+with associated operations. This presentation of operators is simpler and more+flexible than hard-coding them into the syntax tree.+-}++module Language.Python.Syntax.Operator.Binary where++import Control.Lens.Getter ((^.))+import Control.Lens.Lens (lens)+import Control.Lens.TH (makeLenses)+import Data.Functor (($>))+import Data.Semigroup ((<>))++import Language.Python.Syntax.Whitespace++-- | A Python binary operator, such as @+@, along with its trailing 'Whitespace'+--+-- The type variable allows annotations, but it can simply be made @()@ for an unannotated @BinOp@.+data BinOp a+  -- | @a is b@+  = Is a [Whitespace]+  -- | @a is not b@+  | IsNot a [Whitespace] [Whitespace]+  -- | @a in b@+  | In a [Whitespace]+  -- | @a not in b@+  | NotIn a [Whitespace] [Whitespace]+  -- | @a - b@+  | Minus a [Whitespace]+  -- | @a ** b@+  | Exp a [Whitespace]+  -- | @a and b@+  | BoolAnd a [Whitespace]+  -- | @a or b@+  | BoolOr a [Whitespace]+  -- | @a == b@+  | Eq a [Whitespace]+  -- | @a < b@+  | Lt a [Whitespace]+  -- | @a <= b@+  | LtEq a [Whitespace]+  -- | @a > b@+  | Gt a [Whitespace]+  -- | @a >= b@+  | GtEq a [Whitespace]+  -- | @a != b@+  | NotEq a [Whitespace]+  -- | @a * b@+  | Multiply a [Whitespace]+  -- | @a / b@+  | Divide a [Whitespace]+  -- | @a // b@+  | FloorDivide a [Whitespace]+  -- | @a % b@+  | Percent a [Whitespace]+  -- | @a + b@+  | Plus a [Whitespace]+  -- | @a | b@+  | BitOr a [Whitespace]+  -- | @a ^ b@+  | BitXor a [Whitespace]+  -- | @a & b@+  | BitAnd a [Whitespace]+  -- | @a << b@+  | ShiftLeft a [Whitespace]+  -- | @a >> b@+  | ShiftRight a [Whitespace]+  -- | @a @ b@+  | At a [Whitespace]+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (BinOp a) where+  trailingWhitespace =+    lens+      (\case+         Is _ a -> a+         IsNot _ _ a -> a+         In _ a -> a+         NotIn _ _ a -> a+         Minus _ a -> a+         Exp _ a -> a+         BoolAnd _ a -> a+         BoolOr _ a -> a+         Multiply _ a -> a+         Divide _ a -> a+         FloorDivide _ a -> a+         Plus _ a -> a+         Eq _ a -> a+         Lt _ a -> a+         LtEq _ a -> a+         Gt _ a -> a+         GtEq _ a -> a+         NotEq _ a -> a+         BitOr _ a -> a+         BitXor _ a -> a+         BitAnd _ a -> a+         ShiftLeft _ a -> a+         ShiftRight _ a -> a+         Percent _ a -> a+         At _ a -> a)+      (\op ws ->+         case op of+           Is a _ -> Is a ws+           IsNot a b _ -> IsNot a b ws+           In a _ -> In a ws+           NotIn a b _ -> NotIn a b ws+           Minus a _ -> Minus a ws+           Exp a _ -> Exp a ws+           BoolAnd a _ -> BoolAnd a ws+           BoolOr a _ -> BoolOr a ws+           Multiply a _ -> Multiply a ws+           Divide a _ -> Divide a ws+           FloorDivide a _ -> FloorDivide a ws+           Plus a _ -> Plus a ws+           Eq a _ -> Eq a ws+           Lt a _ -> Lt a ws+           LtEq a _ -> LtEq a ws+           Gt a _ -> Gt a ws+           GtEq a _ -> GtEq a ws+           NotEq a _ -> NotEq a ws+           BitOr a _ -> BitOr a ws+           BitAnd a _ -> BitAnd a ws+           BitXor a _ -> BitXor a ws+           ShiftLeft a _ -> ShiftLeft a ws+           ShiftRight a _ -> ShiftRight a ws+           Percent a _ -> Eq a ws+           At a _ -> At a ws)++-- | The associativity of an operator. Each operator is either left-associative or right associative.+--+-- Left associative:+--+-- @+-- x + y + z = (x + y) + z+-- @+--+-- Right associative:+--+-- @+-- x + y + z = x + (y + z)+-- @+data Assoc = L | R deriving (Eq, Show)++-- | An operator along with its precedence and associativity.+data OpEntry+  = OpEntry+  { _opOperator :: BinOp ()+  , _opPrec :: Int+  , _opAssoc :: Assoc+  }+makeLenses ''OpEntry++-- | 'operatorTable' is a list of all operators in ascending order of precedence.+operatorTable :: [OpEntry]+operatorTable =+  [ entry BoolOr 4 L+  , entry BoolAnd 5 L+  , entry Is 10 L+  , entry1 IsNot 10 L+  , entry In 10 L+  , entry1 NotIn 10 L+  , entry Eq 10 L+  , entry Lt 10 L+  , entry LtEq 10 L+  , entry Gt 10 L+  , entry GtEq 10 L+  , entry NotEq 10 L+  , entry BitOr 14 L+  , entry BitXor 15 L+  , entry BitAnd 16 L+  , entry ShiftLeft 17 L+  , entry ShiftRight 17 L+  , entry Minus 20 L+  , entry Plus 20 L+  , entry Multiply 25 L+  , entry At 25 L+  , entry Divide 25 L+  , entry FloorDivide 25 L+  , entry Percent 25 L+  , entry Exp 30 R+  ]+  where+    entry a = OpEntry (a () [])+    entry1 a = OpEntry (a () [] [])++-- | Compare two 'BinOp's to determine whether they represent the same operator, ignoring annotations and trailing whitespace.+sameOperator :: BinOp a -> BinOp a' -> Bool+sameOperator op op' =+  case (op, op') of+    (BoolOr{}, BoolOr{}) -> True+    (BoolAnd{}, BoolAnd{}) -> True+    (Is{}, Is{}) -> True+    (IsNot{}, IsNot{}) -> True+    (In{}, In{}) -> True+    (NotIn{}, NotIn{}) -> True+    (Eq{}, Eq{}) -> True+    (Lt{}, Lt{}) -> True+    (LtEq{}, LtEq{}) -> True+    (Gt{}, Gt{}) -> True+    (GtEq{}, GtEq{}) -> True+    (NotEq{}, NotEq{}) -> True+    (Minus{}, Minus{}) -> True+    (Plus{}, Plus{}) -> True+    (Multiply{}, Multiply{}) -> True+    (Divide{}, Divide{}) -> True+    (FloorDivide{}, FloorDivide{}) -> True+    (Exp{}, Exp{}) -> True+    (Percent{}, Percent{}) -> True+    (BitOr{}, BitOr{}) -> True+    (BitXor{}, BitXor{}) -> True+    (BitAnd{}, BitAnd{}) -> True+    (ShiftLeft{}, ShiftLeft{}) -> True+    (ShiftRight{}, ShiftRight{}) -> True+    (At{}, At{}) -> True+    _ -> False++-- | Is a 'BinOp' a comparison, such as @<=@+isComparison :: BinOp a -> Bool+isComparison a =+  case a of+    Is{} -> True+    IsNot{} -> True+    In{} -> True+    NotIn{} -> True+    Eq{} -> True+    Lt{} -> True+    LtEq{} -> True+    Gt{} -> True+    GtEq{} -> True+    NotEq{} -> True+    _ -> False++-- | Retrieve the information for a given operator from the operator table.+lookupOpEntry :: BinOp a -> [OpEntry] -> OpEntry+lookupOpEntry op =+  go (op $> ())+  where+    go op [] = error $ show op <> " not found in operator table"+    go op (x:xs)+      | sameOperator (x ^. opOperator) op = x+      | otherwise = go op xs
+ src/Language/Python/Syntax/Operator/Unary.hs view
@@ -0,0 +1,42 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language LambdaCase #-}++{-|+Module      : Language.Python.Syntax.Operator.Unary+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Unary operators+-}++module Language.Python.Syntax.Operator.Unary where++import Control.Lens.Lens (lens)+import Language.Python.Syntax.Whitespace++-- | An 'UnOp' is a unary operator in Python, such as @-@ for negation.+-- An operator is stored with an annotation and its trailing whitespace.+data UnOp a+  -- | @-a@+  = Negate a [Whitespace]+  -- | @+a@+  | Positive a [Whitespace]+  -- | @~a@+  | Complement a [Whitespace]+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (UnOp a) where+  trailingWhitespace =+    lens+      (\case+         Negate _ a -> a+         Positive _ a -> a+         Complement _ a -> a)+      (\op ws ->+         case op of+           Negate a _ -> Negate a ws+           Positive a _ -> Positive a ws+           Complement a _ -> Complement a ws)
+ src/Language/Python/Syntax/Punctuation.hs view
@@ -0,0 +1,63 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-|+Module      : Language.Python.Syntax.Punctuation+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++These types are used throughout the syntax tree to help preserve formatting.+-}++module Language.Python.Syntax.Punctuation where++import Control.Lens.Lens (lens)++import Language.Python.Syntax.Whitespace++-- | A period character, possibly followed by some whitespace.+data Dot = MkDot [Whitespace]+  deriving (Eq, Show)++instance HasTrailingWhitespace Dot where+  trailingWhitespace =+    lens (\(MkDot ws) -> ws) (\_ ws -> MkDot ws)++-- | The venerable comma separator+newtype Comma = MkComma [Whitespace]+  deriving (Eq, Show)++instance HasTrailingWhitespace Comma where+  trailingWhitespace =+    lens (\(MkComma ws) -> ws) (\_ ws -> MkComma ws)++newtype Colon = MkColon [Whitespace]+  deriving (Eq, Show)++instance HasTrailingWhitespace Colon where+  trailingWhitespace =+    lens (\(MkColon ws) -> ws) (\_ ws -> MkColon ws)++data Semicolon a = MkSemicolon a [Whitespace]+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (Semicolon a) where+  trailingWhitespace =+    lens (\(MkSemicolon _ ws) -> ws) (\(MkSemicolon a _) ws -> MkSemicolon a ws)++newtype Equals+  = MkEquals [Whitespace]+  deriving (Eq, Show)++instance HasTrailingWhitespace Equals where+  trailingWhitespace =+    lens (\(MkEquals ws) -> ws) (\_ ws -> MkEquals ws)++newtype At+  = MkAt [Whitespace]+  deriving (Eq, Show)++instance HasTrailingWhitespace At where+  trailingWhitespace =+    lens (\(MkAt ws) -> ws) (\_ ws -> MkAt ws)
+ src/Language/Python/Syntax/Raw.hs view
@@ -0,0 +1,5 @@+{-# language DataKinds #-}+module Language.Python.Syntax.Raw where++-- | 'Raw' represents unvalidated, un-annotated terms. +type Raw f = f '[] ()
+ src/Language/Python/Syntax/Statement.hs view
@@ -0,0 +1,702 @@+{-# language TemplateHaskell #-}+{-# language DataKinds, KindSignatures #-}+{-# language MultiParamTypeClasses, FlexibleInstances #-}+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable, DeriveGeneric #-}+{-# language TypeFamilies #-}+{-# language LambdaCase #-}+{-# language UndecidableInstances #-}++{-|+Module      : Language.Python.Syntax.Statement+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.Statement+  ( -- * Statements+    Statement(..)+    -- ** Traversals+  , HasStatements(..)+    -- * Decorators+  , Decorator(..)+    -- ** Compound statements+  , CompoundStatement(..)+    -- ** Small statements+  , SmallStatement(..)+    -- ** Simple statements+  , SimpleStatement(..)+    -- *** @with ... as ...@+  , WithItem(..)+    -- **** Lenses+  , withItemAnn+  , withItemValue+  , withItemBinder+    -- *** @except ... as ...@+  , ExceptAs (..)+    -- **** Lenses+  , exceptAsAnn+  , exceptAsExpr+  , exceptAsName+    -- * Suites+  , Suite(..)+    -- * Blocks+  , Block(..)+    -- ** Lenses+  , blockBlankLines+  , blockHead+  , blockTail+    -- ** Traversals+  , HasBlocks(..)+  )+where++import Control.Lens.Cons (_last)+import Control.Lens.Fold (foldMapOf, folded)+import Control.Lens.Getter ((^.), to, view)+import Control.Lens.Plated (Plated(..), gplate)+import Control.Lens.Prism (_Right)+import Control.Lens.Setter ((.~), over, mapped)+import Control.Lens.TH (makeLenses)+import Control.Lens.Traversal (Traversal, traverseOf)+import Control.Lens.Tuple (_1, _2, _3, _4)+import Data.Bifoldable (bifoldMap)+import Data.Bifunctor (bimap)+import Data.Bitraversable (bitraverse)+import Data.Coerce (coerce)+import Data.List.NonEmpty (NonEmpty)+import Data.Maybe (isNothing)+import Data.Monoid ((<>))+import GHC.Generics (Generic)+import Unsafe.Coerce (unsafeCoerce)++import qualified Data.List.NonEmpty as NonEmpty++import Language.Python.Optics.Validated+import Language.Python.Syntax.AugAssign+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Comment+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Import+import Language.Python.Syntax.ModuleNames+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Whitespace++-- See note [unsafeCoerce Validation] in Language.Python.Internal.Syntax.Expr+instance Validated Statement where; unvalidated = to unsafeCoerce+instance Validated SmallStatement where; unvalidated = to unsafeCoerce+instance Validated SimpleStatement where; unvalidated = to unsafeCoerce+instance Validated CompoundStatement where; unvalidated = to unsafeCoerce+instance Validated Block where; unvalidated = to unsafeCoerce+instance Validated Suite where; unvalidated = to unsafeCoerce+instance Validated WithItem where; unvalidated = to unsafeCoerce+instance Validated ExceptAs where; unvalidated = to unsafeCoerce+instance Validated Decorator where; unvalidated = to unsafeCoerce++-- | 'Traversal' over all the 'Statement's in a term+class HasStatements s where+  _Statements :: Traversal (s v a) (s '[] a) (Statement v a) (Statement '[] a)++-- | A 'Block' is an indented multi-line chunk of code, forming part of a+-- 'Suite'.+data Block (v :: [*]) a+  = Block+  { _blockBlankLines :: [(Blank a, Newline)] -- ^ Blank lines at the beginning of the block+  , _blockHead :: Statement v a -- ^ The first statement of the block+  , _blockTail :: [Either (Blank a, Newline) (Statement v a)] -- ^ The remaining items of the block, which may be statements or blank lines+  } deriving (Eq, Show)++instance Functor (Block v) where+  fmap f (Block a b c) =+    Block+      (over (mapped._1.mapped) f a)+      (fmap f b)+      (bimap (over (_1.mapped) f) (fmap f) <$> c)++instance Foldable (Block v) where+  foldMap f (Block a b c) =+    foldMapOf (folded._1.folded) f a <>+    foldMap f b <>+    foldMap (bifoldMap (foldMapOf (_1.folded) f) (foldMap f)) c++instance Traversable (Block v) where+  traverse f (Block a b c) =+    Block <$>+    traverseOf (traverse._1.traverse) f a <*>+    traverse f b <*>+    traverse (bitraverse (traverseOf (_1.traverse) f) (traverse f)) c++class HasBlocks s where+  -- | 'Traversal' targeting all the 'Block's in a structure+  _Blocks :: Traversal (s v a) (s '[] a) (Block v a) (Block '[] a)++instance HasBlocks Suite where+  _Blocks _ (SuiteOne a b c) = pure $ SuiteOne a b (c ^. unvalidated)+  _Blocks f (SuiteMany a b c d e) = SuiteMany a b c d <$> f e++instance HasBlocks CompoundStatement where+  _Blocks f (Fundef a decos idnt asyncWs ws1 name ws2 params ws3 mty s) =+    Fundef a+      (view unvalidated <$> decos) idnt asyncWs ws1 (coerce name) ws2+      (view unvalidated <$> params) ws3 (over (mapped._2) (view unvalidated) mty) <$>+    _Blocks f s+  _Blocks f (If idnt a ws1 e1 s elifs b') =+    If idnt a ws1 (e1 ^. unvalidated) <$>+    _Blocks f s <*>+    traverse (\(a, b, c, d) -> (,,,) a b (c ^. unvalidated) <$> _Blocks f d) elifs <*>+    traverseOf (traverse._3._Blocks) f b'+  _Blocks f (While idnt a ws1 e1 s els) =+    While idnt a ws1 (e1 ^. unvalidated) <$>+    _Blocks f s <*>+    traverseOf (traverse._3._Blocks) f els+  _Blocks fun (TryExcept idnt a b c d e f) =+    TryExcept idnt a (coerce b) <$>+    _Blocks fun c <*>+    traverse+      (\(a, b, c, d) -> (,,,) a b (view unvalidated <$> c) <$> _Blocks fun d)+      d <*>+    traverseOf (traverse._3._Blocks) fun e <*>+    traverseOf (traverse._3._Blocks) fun f+  _Blocks fun (TryFinally idnt a b c d e f) =+    TryFinally idnt a b <$>+    _Blocks fun c <*>+    pure d <*>+    pure e <*>+    _Blocks fun f+  _Blocks fun (For idnt a asyncWs b c d e f g) =+    For idnt a asyncWs b (c ^. unvalidated) d (view unvalidated <$> e) <$>+    _Blocks fun f <*>+    (traverse._3._Blocks) fun g+  _Blocks fun (ClassDef a decos idnt b c d e) =+    ClassDef a+      (view unvalidated <$> decos) idnt b+      (coerce c) (over (mapped._2.mapped.mapped) (view unvalidated) d) <$>+    _Blocks fun e+  _Blocks fun (With a b asyncWs c d e) =+    With a b asyncWs c (view unvalidated <$> d) <$> _Blocks fun e++instance HasStatements Block where+  _Statements f (Block a b c) =+    Block a <$> f b <*> (traverse._Right) f c++instance HasStatements Suite where+  _Statements _ (SuiteOne a b c) = pure $ SuiteOne a b (c ^. unvalidated)+  _Statements f (SuiteMany a b c d e) = SuiteMany a b c d <$> _Statements f e++-- | See @simpl_stmt@ at <https://docs.python.org/3.5/reference/grammar.html>. The grammar+-- has the terminology mixed up - it should really be called @small_stmt@ there.+data SmallStatement (v :: [*]) a+  = MkSmallStatement+      (SimpleStatement v a)+      [(Semicolon a, SimpleStatement v a)]+      (Maybe (Semicolon a))+      (Maybe (Comment a))+      (Maybe Newline)+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | A 'Statement' is either a 'SmallStatement' or a 'CompoundStatement'+--+-- https://docs.python.org/3.5/reference/compound_stmts.html#compound-statements+data Statement (v :: [*]) a+  = SmallStatement (Indents a) (SmallStatement v a)+  | CompoundStatement (CompoundStatement v a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasExprs SmallStatement where+  _Exprs f (MkSmallStatement s ss a b c) =+    MkSmallStatement <$>+    _Exprs f s <*>+    (traverse._2._Exprs) f ss <*>+    pure a <*>+    pure b <*>+    pure c++instance HasExprs Statement where+  _Exprs f (SmallStatement idnt a) = SmallStatement idnt <$> _Exprs f a+  _Exprs f (CompoundStatement c) = CompoundStatement <$> _Exprs f c++instance HasBlocks SmallStatement where+  _Blocks _ (MkSmallStatement a b c d e) =+    pure $+    MkSmallStatement+      (a ^. unvalidated)+      (over (mapped._2) (view unvalidated) b)+      c d e++instance HasBlocks Statement where+  _Blocks f (CompoundStatement c) = CompoundStatement <$> _Blocks f c+  _Blocks f (SmallStatement idnt a) = SmallStatement idnt <$> _Blocks f a++instance Plated (Statement '[] a) where+  plate _ (SmallStatement idnt s) = pure $ SmallStatement idnt s+  plate fun (CompoundStatement s) =+    CompoundStatement <$>+    case s of+      Fundef idnt a decos asyncWs ws1 b ws2 c ws3 mty s ->+        Fundef idnt a decos asyncWs ws1 b ws2 c ws3 mty <$> _Statements fun s+      If idnt a ws1 b s elifs sts' ->+        If idnt a ws1 b <$>+        _Statements fun s <*>+        (traverse._4._Statements) fun elifs <*>+        (traverse._3._Statements) fun sts'+      While idnt a ws1 b s els ->+        While idnt a ws1 b <$>+        _Statements fun s <*>+        (traverse._3._Statements) fun els+      TryExcept idnt a b c d e f ->+        TryExcept idnt a b <$> _Statements fun c <*>+        (traverse._4._Statements) fun d <*>+        (traverse._3._Statements) fun e <*>+        (traverse._3._Statements) fun f+      TryFinally idnt a b c d e f ->+        TryFinally idnt a b <$> _Statements fun c <*> pure d <*>+        pure e <*> _Statements fun f+      For idnt a asyncWs b c d e f g ->+        For idnt a asyncWs b c d e <$>+        _Statements fun f <*>+        (traverse._3._Statements) fun g+      ClassDef idnt a decos b c d e ->+        ClassDef idnt a decos b c d <$> _Statements fun e+      With a b asyncWs c d e -> With a b asyncWs c (coerce d) <$> _Statements fun e++-- | https://docs.python.org/3.5/reference/simple_stmts.html+data SimpleStatement (v :: [*]) a+  -- | @\'return\' \<spaces\> [\<expr\>]@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-return-statement+  = Return a [Whitespace] (Maybe (Expr v a))+  -- | @\<expr\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#expression-statements+  | Expr a (Expr v a)+  -- | @\<expr\> (\'=\' \<spaces\> \<expr\>)+@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#assignment-statements+  | Assign a (Expr v a) (NonEmpty (Equals, Expr v a))+  -- | @\<expr\> \<augassign\> \<expr\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#augmented-assignment-statements+  | AugAssign a (Expr v a) (AugAssign a) (Expr v a)+  -- | @\'pass\' \<spaces\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-pass-statement+  | Pass a [Whitespace]+  -- | @\'break\' \<spaces\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-break-statement+  | Break a [Whitespace]+  -- | @\'continue\' \<spaces\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-continue-statement+  | Continue a [Whitespace]+  -- | @\'global\' \<spaces\> \<idents\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-global-statement+  | Global a (NonEmpty Whitespace) (CommaSep1 (Ident v a))+  -- | @\'nonlocal\' \<spaces\> \<idents\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-nonlocal-statement+  | Nonlocal a (NonEmpty Whitespace) (CommaSep1 (Ident v a))+  -- | @\'del\' \<spaces\> \<exprs\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-del-statement+  | Del a [Whitespace] (CommaSep1' (Expr v a))+  -- | @\'import\' \<spaces\> \<modulenames\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-import-statement+  | Import+      a+      (NonEmpty Whitespace)+      (CommaSep1 (ImportAs (ModuleName v) v a))+  -- | @\'from\' \<spaces\> \<relative_module_name\> \'import\' \<spaces\> \<import_targets\>@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-import-statement+  | From+      a+      [Whitespace]+      (RelativeModuleName v a)+      [Whitespace]+      (ImportTargets v a)+  -- | @\'raise\' \<spaces\> [\<expr\> [\'as\' \<spaces\> \<expr\>]]@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-raise-statement+  | Raise a+      [Whitespace]+      (Maybe (Expr v a, Maybe ([Whitespace], Expr v a)))+  -- | @\'assert\' \<spaces\> \<expr\> [\',\' \<spaces\> \<expr\>]@+  --+  -- https://docs.python.org/3.5/reference/simple_stmts.html#the-assert-statement+  | Assert a+      [Whitespace]+      (Expr v a)+      (Maybe (Comma, Expr v a))+  deriving (Eq, Show, Functor, Foldable, Traversable, Generic)++instance Plated (SimpleStatement '[] a) where; plate = gplate++instance HasExprs SimpleStatement where+  _Exprs f (Assert a b c d) = Assert a b <$> f c <*> traverseOf (traverse._2) f d+  _Exprs f (Raise a ws x) =+    Raise a ws <$>+    traverse+      (\(b, c) -> (,) <$> f b <*> traverseOf (traverse._2) f c)+      x+  _Exprs f (Return a ws e) = Return a ws <$> traverse f e+  _Exprs f (Expr a e) = Expr a <$> f e+  _Exprs f (Assign a e1 es) = Assign a <$> f e1 <*> traverseOf (traverse._2) f es+  _Exprs f (AugAssign a e1 as e2) = AugAssign a <$> f e1 <*> pure as <*> f e2+  _Exprs _ p@Pass{} = pure $ p ^. unvalidated+  _Exprs _ p@Break{} = pure $ p ^. unvalidated+  _Exprs _ p@Continue{} = pure $ p ^. unvalidated+  _Exprs _ p@Global{} = pure $ p ^. unvalidated+  _Exprs _ p@Nonlocal{} = pure $ p ^. unvalidated+  _Exprs _ p@Del{} = pure $ p ^. unvalidated+  _Exprs _ p@Import{} = pure $ p ^. unvalidated+  _Exprs _ p@From{} = pure $ p ^. unvalidated++-- | See <https://docs.python.org/3.5/reference/compound_stmts.html#the-try-statement>+data ExceptAs (v :: [*]) a+  = ExceptAs+  { _exceptAsAnn :: a+  , _exceptAsExpr :: Expr v a -- ^ @\<expr\>@+  , _exceptAsName :: Maybe ([Whitespace], Ident v a) -- ^ @[\'as\' \<ident\>]@+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | A compound statement consists of one or more clauses.+-- A clause consists of a header and a suite.+data Suite (v :: [*]) a+  -- ':' <space> smallStatement+  = SuiteOne a Colon (SmallStatement v a)+  | SuiteMany a+      -- ':' <spaces> [comment] <newline>+      Colon (Maybe (Comment a)) Newline+      -- <block>+      (Block v a)+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | See <https://docs.python.org/3.5/reference/compound_stmts.html#the-with-statement>+data WithItem (v :: [*]) a+  = WithItem+  { _withItemAnn :: a+  , _withItemValue :: Expr v a -- ^ @\<expr\>@+  , _withItemBinder :: Maybe ([Whitespace], Expr v a) -- ^ @[\'as\' \<spaces\> \<expr\>]@+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | Decorators on function definitions+--+-- <https://docs.python.org/3.5/reference/compound_stmts.html#function-definitions>+--+-- <https://docs.python.org/3.5/glossary.html#term-decorator>+data Decorator (v :: [*]) a+  = Decorator+  { _decoratorAnn :: a+  , _decoratorIndents :: Indents a -- ^ Preceding indentation+  , _decoratorAt :: At -- ^ @\'\@\' \<spaces\>@+  , _decoratorExpr :: Expr v a -- ^ @\<expr\>@+  , _decoratorComment :: Maybe (Comment a) -- ^ Trailing comment+  , _decoratorNewline :: Newline -- ^ Trailing newline+  , _decoratorBlankLines :: [(Blank a, Newline)] -- ^ Trailing blank lines+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++-- | See <https://docs.python.org/3.5/reference/compound_stmts.html>+data CompoundStatement (v :: [*]) a+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#function-definitions+  --+  -- https://docs.python.org/3.5/reference/compound_stmts.html#coroutine-function-definition+  = Fundef+  { _csAnn :: a+  , _unsafeCsFundefDecorators :: [Decorator v a] -- ^ Preceding 'Decorator's+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsFundefAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \<spaces\>]@+  , _unsafeCsFundefDef :: NonEmpty Whitespace -- ^ @\'def\' \<spaces\>@+  , _unsafeCsFundefName :: Ident v a -- ^ @\<ident\>@+  , _unsafeCsFundefLeftParen :: [Whitespace] -- ^ @\'(\' \<spaces\>@+  , _unsafeCsFundefParameters :: CommaSep (Param v a) -- ^ @\<parameters\>@+  , _unsafeCsFundefRightParen :: [Whitespace] -- ^ @\')\' \<spaces\>@+  , _unsafeCsFundefReturnType :: Maybe ([Whitespace], Expr v a) -- ^ @[\'->\' \<spaces\> \<expr\>]@+  , _unsafeCsFundefBody :: Suite v a -- ^ @\<suite\>@+  }+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-if-statement+  | If+  { _csAnn :: a+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsIfIf :: [Whitespace] -- ^ @\'if\' \<spaces\>@+  , _unsafeCsIfCond :: Expr v a -- ^ @\<expr\>@+  , _unsafeCsIfBody :: Suite v a -- ^ @\<suite\>@+  , _unsafeCsIfElifs :: [(Indents a, [Whitespace], Expr v a, Suite v a)] -- ^ @(\'elif\' \<spaces\> \<expr\> \<suite\>)*@+  , _unsafeCsIfElse :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  }+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-while-statement+  | While+  { _csAnn :: a+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsWhileWhile :: [Whitespace] -- ^ @\'while\' \<spaces\>@+  , _unsafeCsWhileCond :: Expr v a -- ^ @\<expr\>@+  , _unsafeCsWhileBody :: Suite v a -- ^ @\<suite\>@+  , _unsafeCsWhileElse+    :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  }+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-try-statement+  | TryExcept+  { _csAnn :: a+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsTryExceptTry :: [Whitespace] -- ^ @\'try\' \<spaces\>@+  , _unsafeCsTryExceptBody :: Suite v a -- ^ @\<suite\>@+  , _unsafeCsTryExceptExcepts :: NonEmpty (Indents a, [Whitespace], Maybe (ExceptAs v a), Suite v a) -- ^ @(\'except\' \<spaces\> \<except_as\> \<suite\>)+@+  , _unsafeCsTryExceptElse :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  , _unsafeCsTryExceptFinally :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'finally\' \<spaces\> \<suite\>]@+  }+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-try-statement+  | TryFinally+  { _csAnn :: a+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsTryFinallyTry :: [Whitespace] -- ^ @\'try\' \<spaces\>@+  , _unsafeCsTryFinallyTryBody :: Suite v a -- ^ @\<suite\>@+  , _unsafeCsTryFinallyFinallyIndents :: Indents a+  , _unsafeCsTryFinallyFinally :: [Whitespace] -- ^ @\'finally\' \<spaces\>@+  , _unsafeCsTryFinallyFinallyBody :: Suite v a -- ^ @\<suite\>@+  }+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-for-statement+  --+  -- https://docs.python.org/3.5/reference/compound_stmts.html#the-async-for-statement+  | For+  { _csAnn :: a+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsForAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \<spaces\>]@+  , _unsafeCsForFor :: [Whitespace] -- ^ @\'for\' \<spaces\>@+  , _unsafeCsForBinder :: Expr v a -- ^ @\<expr\>@+  , _unsafeCsForIn :: [Whitespace] -- ^ @\'in\' \<spaces\>@+  , _unsafeCsForCollection :: CommaSep1' (Expr v a) -- ^ @\<exprs\>@+  , _unsafeCsForBody :: Suite v a -- ^ @\<suite\>@+  , _unsafeCsForElse :: Maybe (Indents a, [Whitespace], Suite v a) -- ^ @[\'else\' \<spaces\> \<suite\>]@+  }+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#class-definitions+  | ClassDef+  { _csAnn :: a+  , _unsafeCsClassDefDecorators :: [Decorator v a] -- ^ Preceding 'Decorator's+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsClassDefClass :: NonEmpty Whitespace -- ^ @\'class\' \<spaces\>@+  , _unsafeCsClassDefName :: Ident v a -- ^ @\<ident\>@+  , _unsafeCsClassDefArguments :: Maybe ([Whitespace], Maybe (CommaSep1' (Arg v a)), [Whitespace]) -- ^ @[\'(\' \<spaces\> [\<args\>] \')\' \<spaces\>]@+  , _unsafeCsClassDefBody :: Suite v a -- ^ @\<suite\>@+  }+  -- | https://docs.python.org/3.5/reference/compound_stmts.html#the-with-statement+  --+  -- https://docs.python.org/3.5/reference/compound_stmts.html#the-async-with-statement+  | With+  { _csAnn :: a+  , _csIndents :: Indents a -- ^ Preceding indentation+  , _unsafeCsWithAsync :: Maybe (NonEmpty Whitespace) -- ^ @[\'async\' \<spaces\>]@+  , _unsafeCsWithWith :: [Whitespace] -- ^ @\'with\' \<spaces\>@+  , _unsafeCsWithItems :: CommaSep1 (WithItem v a) -- ^ @\<with_items\>@+  , _unsafeCsWithBody :: Suite v a -- ^ @\<suite\>@+  }+  deriving (Eq, Show, Functor, Foldable, Traversable)++instance HasExprs ExceptAs where+  _Exprs f (ExceptAs ann e a) = ExceptAs ann <$> f e <*> pure (coerce a)++instance HasExprs Block where+  _Exprs f (Block a b c) =+    Block a <$> _Exprs f b <*> (traverse._Right._Exprs) f c++instance HasExprs Suite where+  _Exprs f (SuiteOne a b c) = (\c' -> SuiteOne a b c') <$> _Exprs f c+  _Exprs f (SuiteMany a b c d e) = SuiteMany a b c d <$> _Exprs f e++instance HasExprs WithItem where+  _Exprs f (WithItem a b c) = WithItem a <$> f b <*> traverseOf (traverse._2) f c++instance HasExprs Decorator where+  _Exprs fun (Decorator a b c d e f g) = (\d' -> Decorator a b c d' e f g) <$> _Exprs fun d++instance HasExprs CompoundStatement where+  _Exprs f (Fundef a decos idnt asyncWs ws1 name ws2 params ws3 mty s) =+    Fundef a <$>+    (traverse._Exprs) f decos <*>+    pure idnt <*>+    pure asyncWs <*>+    pure ws1 <*>+    pure (coerce name) <*>+    pure ws2 <*>+    (traverse._Exprs) f params <*>+    pure ws3 <*>+    traverseOf (traverse._2) f mty <*>+    _Exprs f s+  _Exprs fun (If idnt a ws1 e s elifs sts') =+    If idnt a ws1 <$>+    fun e <*>+    _Exprs fun s <*>+    traverse+      (\(a, b, c, d) -> (,,,) a b <$> fun c <*> _Exprs fun d)+      elifs <*>+    (traverse._3._Exprs) fun sts'+  _Exprs f (While idnt a ws1 e s els) =+    While idnt a ws1 <$>+    f e <*>+    _Exprs f s <*>+    (traverse._3._Exprs) f els+  _Exprs fun (TryExcept idnt a b c d e f) =+    TryExcept idnt a b <$> _Exprs fun c <*>+    traverse+      (\(a, b, c, d) -> (,,,) a b <$> traverse (_Exprs fun) c <*> _Exprs fun d)+      d <*>+    (traverse._3._Exprs) fun e <*>+    (traverse._3._Exprs) fun f+  _Exprs fun (TryFinally idnt a b c d e f) =+    TryFinally idnt a b <$> _Exprs fun c <*> pure d <*>+    pure e <*> _Exprs fun f+  _Exprs fun (For idnt a asyncWs b c d e f g) =+    For idnt a asyncWs b <$> fun c <*> pure d <*> traverse fun e <*>+    _Exprs fun f <*>+    (traverse._3._Exprs) fun g+  _Exprs fun (ClassDef a decos idnt b c d e) =+    ClassDef a <$>+    traverse (_Exprs fun) decos <*>+    pure idnt <*>+    pure b <*>+    pure (coerce c) <*>+    (traverse._2.traverse.traverse._Exprs) fun d <*>+    _Exprs fun e+  _Exprs fun (With a b asyncWs c d e) =+    With a b asyncWs c <$> traverseOf (traverse._Exprs) fun d <*> _Exprs fun e++instance HasTrailingNewline Statement where+  trailingNewline f x =+    case x of+      SmallStatement a b -> SmallStatement a <$> trailingNewline f b+      CompoundStatement c -> CompoundStatement <$> trailingNewline f c++  setTrailingNewline s n =+    case s of+      SmallStatement i a -> SmallStatement i $ setTrailingNewline a n+      CompoundStatement c -> CompoundStatement $ setTrailingNewline c n++instance HasTrailingNewline SmallStatement where+  trailingNewline f (MkSmallStatement a b c d e) =+    MkSmallStatement a b c d <$> traverse f e+  setTrailingNewline (MkSmallStatement a b c d _) n =+    MkSmallStatement a b c d (Just n)++instance HasTrailingNewline Suite where+  trailingNewline f x =+    case x of+      SuiteOne a b c -> SuiteOne a b <$> trailingNewline f c+      SuiteMany a b c d e -> SuiteMany a b c d <$> trailingNewline f e+  setTrailingNewline x n =+    case x of+      SuiteOne a b c -> SuiteOne a b $ setTrailingNewline c n+      SuiteMany a b c d e -> SuiteMany a b c d $ setTrailingNewline e n++instance HasTrailingNewline Block where+  trailingNewline f (Block a b []) = Block a <$> trailingNewline f b <*> pure []+  trailingNewline f (Block a b (c:cs)) =+    Block a b <$>+    traverseOf+      _last+      (bitraverse (traverseOf _2 f) (trailingNewline f))+      (c:cs)++  setTrailingNewline (Block a b []) n =+    Block a (setTrailingNewline b n) []+  setTrailingNewline (Block a b (c:cs)) n =+    Block a b (over _last (bimap (_2 .~ n) (flip setTrailingNewline n)) $ c:cs)++instance HasTrailingNewline CompoundStatement where+  trailingNewline fun s =+    case s of+      Fundef a b c d e f g h i j k ->+        Fundef a b c d e f g h i j <$> trailingNewline fun k+      If a b c d e f g ->+        If a b c d <$>+        (if null f && isNothing g+         then trailingNewline fun e+         else pure e) <*>+        (if isNothing g+         then (_last._4.trailingNewline) fun f+         else pure f)<*>+        (traverse._3.trailingNewline) fun g+      While a b c d e f ->+        While a b c d <$>+        (if isNothing f+         then trailingNewline fun e+         else pure e) <*>+        (traverse._3.trailingNewline) fun f+      TryExcept a b c d e f g ->+        TryExcept a b c d <$>+        (if isNothing f && isNothing g+         then+           fmap+             NonEmpty.fromList+             ((_last._4.trailingNewline) fun $ NonEmpty.toList e)+         else pure e) <*>+        (if isNothing g+         then (traverse._3.trailingNewline) fun f+         else pure f) <*>+        (traverse._3.trailingNewline) fun g+      TryFinally a b c d e f g ->+        TryFinally a b c d e f <$>+        trailingNewline fun g+      For a b c d e f g h i ->+        For a b c d e f g <$>+        (if isNothing i+         then trailingNewline fun h+         else pure h) <*>+        (traverse._3.trailingNewline) fun i+      ClassDef a b c d e f g ->+        ClassDef a b c d e f <$> trailingNewline fun g+      With a b c d e f ->+        With a b c d e <$> trailingNewline fun f+  setTrailingNewline s n =+    case s of+      Fundef a b c d e f g h i j k ->+        Fundef a b c d e f g h i j $ setTrailingNewline k n+      If a b c d e f g ->+        If a b c d+        (if null f && isNothing g+         then setTrailingNewline e n+         else e)+        (if isNothing g+         then over (_last._4) (flip setTrailingNewline n) f+         else f)+        (over (mapped._3) (flip setTrailingNewline n) g)+      While a b c d e f ->+        While a b c d+        (if isNothing f+         then setTrailingNewline e n+         else e)+        (over (mapped._3) (flip setTrailingNewline n) f)+      TryExcept a b c d e f g ->+        TryExcept a b c d+        (if isNothing f && isNothing g+         then+             NonEmpty.fromList+             (over (_last._4) (flip setTrailingNewline n) $ NonEmpty.toList e)+         else e)+        (if isNothing g+         then over (mapped._3) (flip setTrailingNewline n) f+         else f)+        (over (mapped._3) (flip setTrailingNewline n) g)+      TryFinally a b c d e f g ->+        TryFinally a b c d e f $+        setTrailingNewline g n+      For a b c d e f g h i ->+        For a b c d e f g+        (if isNothing i+         then setTrailingNewline h n+         else h)+        (over (mapped._3) (flip setTrailingNewline n) i)+      ClassDef a b c d e f g ->+        ClassDef a b c d e f $ setTrailingNewline g n+      With a b c d e f ->+        With a b c d e $ setTrailingNewline f n++makeLenses ''WithItem+makeLenses ''ExceptAs+makeLenses ''Block
+ src/Language/Python/Syntax/Strings.hs view
@@ -0,0 +1,318 @@+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language OverloadedStrings #-}+{-# language LambdaCase #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Syntax.Strings+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Python string literals.++See <https://docs.python.org/3.5/reference/lexical_analysis.html#string-and-bytes-literals>+-}++module Language.Python.Syntax.Strings+  ( -- * Datatypes+    -- ** Characters+    PyChar(..)+  , fromHaskellString+    -- ** String information+  , QuoteType(..)+  , StringType(..)+    -- ** String prefixes+  , StringPrefix(..)+  , RawStringPrefix(..)+  , BytesPrefix(..)+  , RawBytesPrefix(..)+  , hasPrefix+    -- ** String literals+  , StringLiteral(..)+    -- *** Lenses+  , stringLiteralStringType+  , stringLiteralQuoteType+  , stringLiteralValue+  , stringLiteralWhitespace+    -- * Rendering+  , showQuoteType+  , showStringPrefix+  , showRawStringPrefix+  , showBytesPrefix+  , showRawBytesPrefix+    -- * Extra functions+  , isEscape+  )+where++import Control.Lens.Lens (lens)+import Control.Lens.TH (makeLensesFor)+import Data.Digit.Octal (OctDigit)+import Data.Digit.Hexadecimal.MixedCase (HeXDigit(..))+import Data.Maybe (isJust)+import Data.Text (Text)++import Language.Python.Syntax.Whitespace++-- | Double or single quotation marks?+--+-- @+-- "Double quotes"+-- """Double quotes"""+-- 'Single quotes'+-- '''Single quotes'''+-- @+data QuoteType+  = SingleQuote+  | DoubleQuote+  deriving (Eq, Ord, Show)++-- | Three pairs of quotations or one?+--+-- @+-- """Long string"""+-- '''Also long string'''+-- "Short string"+-- 'Also short string'+-- @+data StringType+  = ShortString+  | LongString+  deriving (Eq, Ord, Show)++-- | In Python 3.5, a prefix of @u@ or @U@ is allowed, but doesn't have any+-- meaning. They exist for backwards compatibility with Python 2.+--+-- See <https://www.python.org/dev/peps/pep-0414/>+data StringPrefix+  = Prefix_u+  | Prefix_U+  deriving (Eq, Ord, Show)++-- | Raw strings are prefixed with either @r@ or @R@.+data RawStringPrefix+  = Prefix_r+  | Prefix_R+  deriving (Eq, Ord, Show)++-- | This prefix indicates it's a bytes literal rather than a string literal.+data BytesPrefix+  = Prefix_b+  | Prefix_B+  deriving (Eq, Ord, Show)++-- | A string of raw bytes can be indicated by a number of prefixes+data RawBytesPrefix+  = Prefix_br+  | Prefix_Br+  | Prefix_bR+  | Prefix_BR+  | Prefix_rb+  | Prefix_rB+  | Prefix_Rb+  | Prefix_RB+  deriving (Eq, Ord, Show)++-- | Most types of 'StringLiteral' have prefixes. Plain old strings may have+-- an optional prefix, but it is meaningless.+hasPrefix :: StringLiteral a -> Bool+hasPrefix RawStringLiteral{} = True+hasPrefix RawBytesLiteral{} = True+hasPrefix (StringLiteral _ a _ _ _ _) = isJust a+hasPrefix BytesLiteral{} = True++-- | A 'StringLiteral', complete with a prefix, information about+-- quote type and number, and a list of 'PyChar's.+--+-- Like many other data types in hpython, it has an annotation and+-- trailing whitespace.+data StringLiteral a+  = RawStringLiteral+  { _stringLiteralAnn :: a+  , _unsafeRawStringLiteralPrefix :: RawStringPrefix+  , _stringLiteralStringType :: StringType+  , _stringLiteralQuoteType :: QuoteType+  , _stringLiteralValue :: [PyChar]+  , _stringLiteralWhitespace :: [Whitespace]+  }+  | StringLiteral+  { _stringLiteralAnn :: a+  , _unsafeStringLiteralPrefix :: Maybe StringPrefix+  , _stringLiteralStringType :: StringType+  , _stringLiteralQuoteType :: QuoteType+  , _stringLiteralValue :: [PyChar]+  , _stringLiteralWhitespace :: [Whitespace]+  }+  | RawBytesLiteral+  { _stringLiteralAnn :: a+  , _unsafeRawBytesLiteralPrefix :: RawBytesPrefix+  , _stringLiteralStringType :: StringType+  , _stringLiteralQuoteType :: QuoteType+  , _stringLiteralValue :: [PyChar]+  , _stringLiteralWhitespace :: [Whitespace]+  }+  | BytesLiteral+  { _stringLiteralAnn :: a+  , _unsafeBytesLiteralPrefix :: BytesPrefix+  , _stringLiteralStringType :: StringType+  , _stringLiteralQuoteType :: QuoteType+  , _stringLiteralValue :: [PyChar]+  , _stringLiteralWhitespace :: [Whitespace]+  }+  deriving (Eq, Ord, Show, Functor, Foldable, Traversable)++instance HasTrailingWhitespace (StringLiteral a) where+  trailingWhitespace =+    lens+      (\case+          RawStringLiteral _ _ _ _ _ ws -> ws+          StringLiteral _ _ _ _ _ ws -> ws+          RawBytesLiteral _ _ _ _ _ ws -> ws+          BytesLiteral _ _ _ _ _ ws -> ws)+      (\s ws -> case s of+          StringLiteral a b c d e _ -> StringLiteral a b c d e ws+          RawStringLiteral a b c d e _ -> RawStringLiteral a b c d e ws+          BytesLiteral a b c d e _ -> BytesLiteral a b c d e ws+          RawBytesLiteral a b c d e _ -> RawBytesLiteral a b c d e ws)++-- | A character in a string literal. This is a large sum type, with a+-- catch-all of a Haskell 'Char'.+data PyChar+  -- | @\\newline@+  = Char_newline+  -- | @\\1@+  | Char_octal1 OctDigit+  -- | @\\12@+  | Char_octal2 OctDigit OctDigit+  -- | @\\123@+  | Char_octal3 OctDigit OctDigit OctDigit+  -- | @\\xFb@+  | Char_hex HeXDigit HeXDigit+  -- | @\\u12aD@+  | Char_uni16+      HeXDigit+      HeXDigit+      HeXDigit+      HeXDigit+  -- | @\\Udeadbeef@+  | Char_uni32+      HeXDigit+      HeXDigit+      HeXDigit+      HeXDigit+      HeXDigit+      HeXDigit+      HeXDigit+      HeXDigit+  -- | @\\\\@+  | Char_esc_bslash+  -- | @\\\'@+  | Char_esc_singlequote+  -- | @\\\"@+  | Char_esc_doublequote+  -- | @\\a@+  | Char_esc_a+  -- | @\\b@+  | Char_esc_b+  -- | @\\f@+  | Char_esc_f+  -- | @\\n@+  | Char_esc_n+  -- | @\\r@+  | Char_esc_r+  -- | @\\t@+  | Char_esc_t+  -- | @\\v@+  | Char_esc_v+  -- | Any character+  | Char_lit Char+  deriving (Eq, Ord, Show)++-- | Determine whether a 'PyChar' is an escape character or not.+isEscape :: PyChar -> Bool+isEscape c =+  case c of+    Char_newline -> True+    Char_octal1{} -> True+    Char_octal2{} -> True+    Char_octal3{} -> True+    Char_hex{} -> True+    Char_uni16{} -> True+    Char_uni32{} -> True+    Char_esc_bslash -> True+    Char_esc_singlequote -> True+    Char_esc_doublequote -> True+    Char_esc_a -> True+    Char_esc_b -> True+    Char_esc_f -> True+    Char_esc_n -> True+    Char_esc_r -> True+    Char_esc_t -> True+    Char_esc_v -> True+    Char_lit{} -> False++-- | Convert a Haskell string to a list of 'PyChar'. This is useful when+-- writing Python in Haskell.+fromHaskellString :: String -> [PyChar]+fromHaskellString =+  fmap+  (\c -> case c of+    '\\' -> Char_esc_bslash+    '\'' -> Char_esc_singlequote+    '\"' -> Char_esc_doublequote+    '\a' -> Char_esc_a+    '\b' -> Char_esc_b+    '\f' -> Char_esc_f+    '\n' -> Char_esc_n+    '\r' -> Char_esc_r+    '\t' -> Char_esc_t+    '\v' -> Char_esc_v+    '\0' -> Char_hex HeXDigit0 HeXDigit0+    _ -> Char_lit c)++showStringPrefix :: StringPrefix -> Text+showStringPrefix sp =+  case sp of+    Prefix_u -> "u"+    Prefix_U -> "U"++showRawStringPrefix :: RawStringPrefix -> Text+showRawStringPrefix sp =+  case sp of+    Prefix_r -> "r"+    Prefix_R -> "R"++showBytesPrefix :: BytesPrefix -> Text+showBytesPrefix sp =+  case sp of+    Prefix_b -> "b"+    Prefix_B -> "B"++showRawBytesPrefix :: RawBytesPrefix -> Text+showRawBytesPrefix sp =+  case sp of+    Prefix_br -> "br"+    Prefix_Br -> "Br"+    Prefix_bR -> "bR"+    Prefix_BR -> "BR"+    Prefix_rb -> "rb"+    Prefix_rB -> "rB"+    Prefix_Rb -> "Rb"+    Prefix_RB -> "RB"++showQuoteType :: QuoteType -> Char+showQuoteType qt =+  case qt of+    DoubleQuote -> '\"'+    SingleQuote -> '\''++makeLensesFor+  [ ("_stringLiteralValue", "stringLiteralValue")+  , ("_stringLiteralStringType", "stringLiteralStringType")+  , ("_stringLiteralQuoteType", "stringLiteralQuoteType")+  , ("_stringLiteralWhitespace", "stringLiteralWhitespace")+  ]+  ''StringLiteral
+ src/Language/Python/Syntax/Types.hs view
@@ -0,0 +1,427 @@+{-# language DataKinds #-}+{-# language KindSignatures #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Syntax.Types+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable++Datatypes for different parts of Python syntax+-}++module Language.Python.Syntax.Types+  ( -- * Parameters+    -- ** Positional parameters+    PositionalParam(..)+    -- *** Lenses+  , ppAnn+  , ppName+  , ppType+    -- ** Starred Parameters+  , StarParam(..)+    -- *** Lenses+  , spAnn+  , spWhitespace+  , spName+  , spType+    -- ** Unnamed Starred Parameters+  , UnnamedStarParam(..)+    -- *** Lenses+  , uspAnn+  , uspWhitespace+    -- ** Keyword parameters+  , KeywordParam(..)+    -- *** Lenses+  , kpAnn+  , kpName+  , kpType+  , kpEquals+  , kpExpr+    -- * Compound statements+    -- ** Function definitions+  , Fundef(..)+    -- *** Lenses+  , fdAnn+  , fdDecorators+  , fdIndents+  , fdAsync+  , fdDefSpaces+  , fdName+  , fdLeftParenSpaces+  , fdParameters+  , fdRightParenSpaces+  , fdReturnType+  , fdBody+    -- ** Class definitions+  , ClassDef(..)+    -- *** Lenses+  , cdAnn+  , cdDecorators+  , cdIndents+  , cdClass+  , cdName+  , cdArguments+  , cdBody+    -- ** @if@ statements+  , If(..)+    -- *** Lenses+  , ifAnn+  , ifIndents+  , ifIf+  , ifCond+  , ifBody+  , ifElifs+  , ifElse+    -- ** @elif@+  , Elif(..)+    -- *** Lenses+  , elifIndents+  , elifElif+  , elifCond+  , elifBody+    -- ** @for@ statements+  , For(..)+    -- *** Lenses+  , forAnn+  , forIndents+  , forAsync+  , forFor+  , forBinder+  , forIn+  , forCollection+  , forBody+  , forElse+    -- ** @while@ statements+  , While(..)+    -- *** Lenses+  , whileAnn+  , whileIndents+  , whileWhile+  , whileCond+  , whileBody+  , whileElse+    -- ** @try ... except ... else ... finally@+  , TryExcept(..)+    -- *** Lenses+  , teAnn+  , teIndents+  , teTry+  , teBody+  , teExcepts+  , teElse+  , teFinally+    -- *** @except@+  , Except(..)+    -- **** Lenses+  , exceptIndents+  , exceptExcept+  , exceptExceptAs+  , exceptBody+    -- ** @try ... finally@+  , TryFinally(..)+    -- *** Lenses+  , tfAnn+  , tfIndents+  , tfTry+  , tfBody+  , tfFinally+    -- ** @finally@+  , Finally(..)+    -- *** Lenses+  , finallyIndents+  , finallyFinally+  , finallyBody+    -- ** @with@ statements+  , With(..)+    -- *** Lenses+  , withAnn+  , withIndents+  , withAsync+  , withWith+  , withItems+  , withBody+    -- ** @else@+  , Else(..)+    -- *** Lenses+  , elseIndents+  , elseElse+  , elseBody+    -- * Expressions+    -- ** @None@+  , None(..)+    -- *** Lenses+  , noneAnn+  , noneWhitespace+    -- ** Function calls+  , Call(..)+    -- *** Lenses+  , callAnn+  , callFunction+  , callLeftParen+  , callArguments+  , callRightParen+    -- ** Tuples+  , Tuple(..)+    -- *** Lenses+  , tupleAnn+  , tupleHead+  , tupleComma+  , tupleTail+    -- *** Tuple items+    -- **** Unpacking+  , TupleUnpack(..)+    -- ***** Lenses+  , tupleUnpackAnn+  , tupleUnpackParens+  , tupleUnpackWhitespace+  , tupleUnpackValue+    -- ** Lists+  , List(..)+    -- *** Lenses+  , listAnn+  , listWhitespaceLeft+  , listBody+  , listWhitespaceRight+    -- *** List items+    -- **** Unpacking+  , ListUnpack(..)+    -- ***** Lenses+  , listUnpackAnn+  , listUnpackParens+  , listUnpackWhitespace+  , listUnpackValue+  )+where++import Control.Lens.TH (makeLenses)+import Data.List.NonEmpty (NonEmpty)++import Language.Python.Syntax.CommaSep (Comma, CommaSep, CommaSep1, CommaSep1')+import Language.Python.Syntax.Expr (Arg, Expr, ListItem, Param, TupleItem)+import Language.Python.Syntax.Ident (Ident)+import Language.Python.Syntax.Punctuation (Colon)+import Language.Python.Syntax.Statement (Decorator, ExceptAs, Suite, WithItem)+import Language.Python.Syntax.Whitespace++data Fundef v a+  = MkFundef+  { _fdAnn :: a+  , _fdDecorators :: [Decorator v a]+  , _fdIndents :: Indents a+  , _fdAsync :: Maybe (NonEmpty Whitespace)+  , _fdDefSpaces :: NonEmpty Whitespace+  , _fdName :: Ident v a+  , _fdLeftParenSpaces :: [Whitespace]+  , _fdParameters :: CommaSep (Param v a)+  , _fdRightParenSpaces :: [Whitespace]+  , _fdReturnType :: Maybe ([Whitespace], Expr v a)+  , _fdBody :: Suite v a+  } deriving (Eq, Show)+makeLenses ''Fundef++data Else v a+  = MkElse+  { _elseIndents :: Indents a+  , _elseElse :: [Whitespace]+  , _elseBody :: Suite v a+  } deriving (Eq, Show)+makeLenses ''Else++data While v a+  = MkWhile+  { _whileAnn :: a+  , _whileIndents :: Indents a+  , _whileWhile :: [Whitespace]+  , _whileCond :: Expr v a+  , _whileBody :: Suite v a+  , _whileElse :: Maybe (Else v a)+  } deriving (Eq, Show)+makeLenses ''While++data KeywordParam v a+  = MkKeywordParam+  { _kpAnn :: a+  , _kpName :: Ident v a+  , _kpType :: Maybe (Colon, Expr v a)+  , _kpEquals :: [Whitespace]+  , _kpExpr :: Expr v a+  } deriving (Eq, Show)+makeLenses ''KeywordParam++data PositionalParam v a+  = MkPositionalParam+  { _ppAnn :: a+  , _ppName :: Ident v a+  , _ppType :: Maybe (Colon, Expr v a)+  } deriving (Eq, Show)+makeLenses ''PositionalParam++data StarParam v a+  = MkStarParam+  { _spAnn :: a+  , _spWhitespace :: [Whitespace]+  , _spName :: Ident v a+  , _spType :: Maybe (Colon, Expr v a)+  } deriving (Eq, Show)+makeLenses ''StarParam++data UnnamedStarParam (v :: [*]) a+  = MkUnnamedStarParam+  { _uspAnn :: a+  , _uspWhitespace :: [Whitespace]+  } deriving (Eq, Show)+makeLenses ''UnnamedStarParam++data Call v a+  = MkCall+  { _callAnn :: a+  , _callFunction :: Expr v a+  , _callLeftParen :: [Whitespace]+  , _callArguments :: Maybe (CommaSep1' (Arg v a))+  , _callRightParen :: [Whitespace]+  } deriving (Eq, Show)+makeLenses ''Call++data Elif v a+  = MkElif+  { _elifIndents :: Indents a+  , _elifElif :: [Whitespace]+  , _elifCond :: Expr v a+  , _elifBody :: Suite v a+  } deriving (Eq, Show)+makeLenses ''Elif++data If v a+  = MkIf+  { _ifAnn :: a+  , _ifIndents :: Indents a+  , _ifIf :: [Whitespace]+  , _ifCond :: Expr v a+  , _ifBody :: Suite v a+  , _ifElifs :: [Elif v a]+  , _ifElse :: Maybe (Else v a)+  } deriving (Eq, Show)+makeLenses ''If++data For v a+  = MkFor+  { _forAnn :: a+  , _forIndents :: Indents a+  , _forAsync :: Maybe (NonEmpty Whitespace)+  , _forFor :: [Whitespace]+  , _forBinder :: Expr v a+  , _forIn :: [Whitespace]+  , _forCollection :: CommaSep1' (Expr v a)+  , _forBody :: Suite v a+  , _forElse :: Maybe (Else v a)+  } deriving (Eq, Show)+makeLenses ''For++data Finally v a+  = MkFinally+  { _finallyIndents :: Indents a+  , _finallyFinally :: [Whitespace]+  , _finallyBody :: Suite v a+  } deriving (Eq, Show)+makeLenses ''Finally++data Except v a+  = MkExcept+  { _exceptIndents :: Indents a+  , _exceptExcept :: [Whitespace]+  , _exceptExceptAs :: Maybe (ExceptAs v a)+  , _exceptBody :: Suite v a+  } deriving (Eq, Show)+makeLenses ''Except++data TryExcept v a+  = MkTryExcept+  { _teAnn :: a+  , _teIndents :: Indents a+  , _teTry :: [Whitespace]+  , _teBody :: Suite v a+  , _teExcepts :: NonEmpty (Except v a)+  , _teElse :: Maybe (Else v a)+  , _teFinally :: Maybe (Finally v a)+  } deriving (Eq, Show)+makeLenses ''TryExcept++data TryFinally v a+  = MkTryFinally+  { _tfAnn :: a+  , _tfIndents :: Indents a+  , _tfTry :: [Whitespace]+  , _tfBody :: Suite v a+  , _tfFinally :: Finally v a+  } deriving (Eq, Show)+makeLenses ''TryFinally++data ClassDef v a+  = MkClassDef+  { _cdAnn :: a+  , _cdDecorators :: [Decorator v a]+  , _cdIndents :: Indents a+  , _cdClass :: NonEmpty Whitespace+  , _cdName :: Ident v a+  , _cdArguments :: Maybe ([Whitespace], Maybe (CommaSep1' (Arg v a)), [Whitespace])+  , _cdBody :: Suite v a+  } deriving (Eq, Show)+makeLenses ''ClassDef++data With v a+  = MkWith+  { _withAnn :: a+  , _withIndents :: Indents a+  , _withAsync :: Maybe (NonEmpty Whitespace)+  , _withWith :: [Whitespace]+  , _withItems :: CommaSep1 (WithItem v a)+  , _withBody :: Suite v a+  } deriving (Eq, Show)+makeLenses ''With++data Tuple v a+  = MkTuple+  { _tupleAnn :: a+  , _tupleHead :: TupleItem v a+  , _tupleComma :: Comma+  , _tupleTail :: Maybe (CommaSep1' (TupleItem v a))+  } deriving (Eq, Show)+makeLenses ''Tuple++data List v a+  = MkList+  { _listAnn :: a+  , _listWhitespaceLeft :: [Whitespace]+  , _listBody :: Maybe (CommaSep1' (ListItem v a))+  , _listWhitespaceRight :: [Whitespace]+  } deriving (Eq, Show)+makeLenses ''List++data ListUnpack v a+  = MkListUnpack+  { _listUnpackAnn :: a+  , _listUnpackParens :: [([Whitespace], [Whitespace])]+  , _listUnpackWhitespace :: [Whitespace]+  , _listUnpackValue :: Expr v a+  } deriving (Eq, Show)+makeLenses ''ListUnpack++data None (v :: [*]) a+  = MkNone+  { _noneAnn :: a+  , _noneWhitespace :: [Whitespace]+  } deriving (Eq, Show)+makeLenses ''None++data TupleUnpack v a+  = MkTupleUnpack+  { _tupleUnpackAnn :: a+  , _tupleUnpackParens :: [([Whitespace], [Whitespace])]+  , _tupleUnpackWhitespace :: [Whitespace]+  , _tupleUnpackValue :: Expr v a+  } deriving (Eq, Show)+makeLenses ''TupleUnpack
+ src/Language/Python/Syntax/Whitespace.hs view
@@ -0,0 +1,205 @@+{-# language DataKinds #-}+{-# language GeneralizedNewtypeDeriving, MultiParamTypeClasses, BangPatterns #-}+{-# language TypeFamilies #-}+{-# language DeriveFunctor, DeriveFoldable, DeriveTraversable #-}+{-# language TemplateHaskell #-}++{-|+Module      : Language.Python.Syntax.Whitespace+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Syntax.Whitespace+  ( -- * Whitespace+    Newline(..)+  , Whitespace(..)+  , Blank(..)+  , HasTrailingWhitespace(..)+  , HasTrailingNewline(..)+    -- * Indentation+  , IndentLevel, getIndentLevel, indentLevel, absoluteIndentLevel+  , Indent(..), indentWhitespaces, indentIt, dedentIt+  , Indents(..), indentsValue, indentsAnn, subtractStart+  )+where++import Control.Lens.Iso (Iso', iso, from)+import Control.Lens.Getter ((^.), view)+import Control.Lens.Lens (Lens', lens)+import Control.Lens.Setter ((.~))+import Control.Lens.TH (makeLenses)+import Control.Lens.Traversal (Traversal')+import Data.Deriving (deriveEq1, deriveOrd1)+import Data.Foldable (toList)+import Data.Function ((&))+import Data.FingerTree (FingerTree, Measured(..), fromList)+import Data.List (stripPrefix)+import Data.List.NonEmpty (NonEmpty(..))+import Data.Monoid (Monoid, Endo(..), Dual(..))+import Data.Semigroup (Semigroup, (<>))+import GHC.Exts (IsList(..))++import qualified Data.List.NonEmpty as NonEmpty++import Language.Python.Syntax.Comment (Comment)++-- | A newline is either a carriage return, a line feed, or a carriage return+-- followed by a line feed.+data Newline = CR | LF | CRLF deriving (Eq, Ord, Show)++-- | Whitespace is either a space, a tab, a newline that continues the+-- logical line ('Continued'), a newline that ends the logical line ('Newline'),+-- or a 'Comment'.+--+-- Despite not literally being whitespace, comments inside enclosed forms+-- are treated as whitespace. See <https://docs.python.org/3.5/reference/lexical_analysis.html#implicit-line-joining>+--+-- Example and counterexample of comments as whitespace+--+-- @+--( 1 ++--  # here's a comment+-- 2 ++-- 3 # another comment+--)+-- @+--+-- @+-- x = 5 + 5+-- # this line is not considered whitespace+-- y = x * 2+-- @+--+-- @+-- [ 1+-- , 2 # I'm whitespace+-- , 3+-- # also whitespace+-- ]+-- @+data Whitespace+  = Space+  | Tab+  | Continued Newline [Whitespace]+  | Newline Newline+  | Comment (Comment ())+  deriving (Eq, Ord, Show)++-- | Every syntactic element contains the whitespace that immediately follows it.+--+-- This type class lets us access this trailing whitespace in many different+-- types throughout hpython.+class HasTrailingWhitespace s where+  trailingWhitespace :: Lens' s [Whitespace]++instance HasTrailingWhitespace a => HasTrailingWhitespace (NonEmpty a) where+  trailingWhitespace =+    lens+      (view trailingWhitespace . NonEmpty.last)+      (\(x :| xs) ws ->+         case xs of+           [] -> (x & trailingWhitespace .~ ws) :| xs+           x' : xs' -> NonEmpty.cons x $ (x' :| xs') & trailingWhitespace .~ ws)++-- | A statement-containing thing may have a trailing newline+--+-- Some forms /always/ have a trailing newline, which is why this class isn't just+-- @trailingNewline :: 'Lens'' (s v a) ('Maybe' 'Newline')@+class HasTrailingNewline (s :: [*] -> * -> *) where+  trailingNewline :: Traversal' (s v a) Newline+  setTrailingNewline :: s v a -> Newline -> s v a++-- | Lines which are "blank", meaning that they contain, if anything, only+-- whitespace and/or a comment.+data Blank a+  = Blank+  { _blankAnn :: a+  , _blankWhitespaces :: [Whitespace]+  , _blankComment :: Maybe (Comment a)+  } deriving (Eq, Show, Functor, Foldable, Traversable)++-- | Python has rules regarding the expansion of tabs into spaces and how to+-- go about computing indentation after this is done.+--+-- See <https://docs.python.org/3.5/reference/lexical_analysis.html#indentation>+--+-- This data structure implements those rules as a monoid.+newtype IndentLevel+  = IndentLevel+  { appIndentLevel+    :: Maybe Int -> Dual (Endo (Bool, Int))+  }+  deriving (Semigroup, Monoid)++indentLevel :: Indent -> Int+indentLevel = getIndentLevel . measure . unIndent++getIndentLevel :: IndentLevel -> Int+getIndentLevel il =+  snd $+  appEndo (getDual (appIndentLevel il Nothing)) (False, 0)++absoluteIndentLevel :: Int -> Indent -> Int+absoluteIndentLevel n il =+  snd $+  appEndo (getDual (appIndentLevel (measure $ unIndent il) $ Just n)) (False, 0)++instance Measured IndentLevel Whitespace where+  measure e =+    IndentLevel $+    \absolute -> Dual . Endo $+    \(b, !i) ->+    case e of+      Space -> (b, if b then i else i+1)+      Tab -> (b, if b then i else maybe (i + 8 - rem i 8) (+i) absolute)+      Continued{} -> (True, i)+      Newline{} -> error "Newline does not have an IndentLevel"+      Comment{} -> error "Comment does not have an IndentLevel"++newtype Indent+  = MkIndent+  { unIndent :: FingerTree IndentLevel Whitespace+  } deriving (Eq, Ord, Show, Semigroup, Monoid)++instance IsList Indent where+  type Item Indent = Whitespace+  toList = view indentWhitespaces+  fromList = view $ from indentWhitespaces++-- | Indent some indentation by a chunk+indentIt :: [Whitespace] -> Indents a -> Indents a+indentIt ws (Indents a b) = Indents (ws ^. from indentWhitespaces : a) b++-- | Deent some indentation by a chunk+dedentIt :: Indents a -> Indents a+dedentIt i@(Indents [] _) = i+dedentIt (Indents (_:b) c) = Indents b c++-- | An 'Indent' is isomorphic to a list of 'Whitespace'+indentWhitespaces :: Iso' Indent [Whitespace]+indentWhitespaces =+  iso (Data.Foldable.toList . unIndent) (MkIndent . Data.FingerTree.fromList)++-- | Subtract the first argument from the beginning of the second+--+-- Returns 'Nothing' if the first list is not a prefix of the second.+subtractStart :: Indents a -> Indents a -> Maybe (Indents a)+subtractStart (Indents a _) (Indents b c) = Indents <$> stripPrefix a b <*> pure c++-- | A possibly annotated list of 'Indent's.+data Indents a+  = Indents+  { _indentsValue :: [Indent]+  , _indentsAnn :: a+  } deriving (Eq, Show, Functor, Foldable, Traversable)++instance Semigroup a => Semigroup (Indents a) where+  Indents a b <> Indents c d = Indents (a <> c) (b <> d)++makeLenses ''Indents+deriveEq1 ''Indents+deriveOrd1 ''Indents
+ src/Language/Python/Validate.hs view
@@ -0,0 +1,88 @@+{-# language DataKinds, TypeOperators #-}+{-# language FlexibleContexts #-}+{-|+Module      : Language.Python.Validate+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Validate+  ( module Data.Validation+  , module Language.Python.Validate.Error+  , module Language.Python.Validate.Indentation+  , module Language.Python.Validate.Scope+  , module Language.Python.Validate.Syntax+  , validateModuleAll+  , validateStatementAll+  , validateExprAll+  , validateAll+  )+where++import Data.List.NonEmpty (NonEmpty)+import Data.Validation++import Language.Python.Syntax+import Language.Python.Validate.Error+import Language.Python.Validate.Indentation+import Language.Python.Validate.Scope+import Language.Python.Validate.Syntax++validateModuleAll+  :: ( AsIndentationError e a+     , AsSyntaxError e a+     , AsScopeError e a+     )+  => Module '[] a -- ^ 'Module' to validate+  -> Validation (NonEmpty e) (Module '[Scope, Syntax, Indentation] a)+validateModuleAll =+  validateAll validateModuleIndentation validateModuleSyntax validateModuleScope++validateStatementAll+  :: ( AsIndentationError e a+     , AsSyntaxError e a+     , AsScopeError e a+     )+  => Statement '[] a -- ^ 'Statement' to validate+  -> Validation (NonEmpty e) (Statement '[Scope, Syntax, Indentation] a)+validateStatementAll =+  validateAll validateStatementIndentation validateStatementSyntax validateStatementScope++validateExprAll+  :: ( AsIndentationError e a+     , AsSyntaxError e a+     , AsScopeError e a+     )+  => Expr '[] a -- ^ 'Expr' to validate+  -> Validation (NonEmpty e) (Expr '[Scope, Syntax, Indentation] a)+validateExprAll =+  validateAll validateExprIndentation validateExprSyntax validateExprScope++-- | Validate a datatype for indentation, syntax, and scope correctness+--+-- e.g.+--+-- @+-- 'validateModuleAll' =+--   'validateAll'+--     'validateModuleIndentation'+--     'validateModuleSyntax'+--     'validateModuleScope'+-- @+validateAll+  :: ( AsIndentationError e a+     , AsSyntaxError e a+     , AsScopeError e a+     )+  => (s '[] a -> ValidateIndentation e (s '[Indentation] a)) -- ^ Indentation validator+  -> (s '[Indentation] a -> ValidateSyntax e (s '[Syntax, Indentation] a)) -- ^ Syntax validator+  -> (s '[Syntax, Indentation] a -> ValidateScope a e (s '[Scope, Syntax, Indentation] a)) -- ^ Scope validator+  -> s '[] a+  -> Validation (NonEmpty e) (s '[Scope, Syntax, Indentation] a)+validateAll vi vsyn vsco m =+  runValidateIndentation (vi m) `bindValidation` \m' ->+  runValidateSyntax (vsyn m') `bindValidation` \m'' ->+  runValidateScope (vsco m'')
+ src/Language/Python/Validate/Error.hs view
@@ -0,0 +1,41 @@+{-# language FlexibleInstances, MultiParamTypeClasses #-}+{-# language LambdaCase #-}+module Language.Python.Validate.Error+  ( module Language.Python.Validate.Indentation.Error+  , module Language.Python.Validate.Scope.Error+  , module Language.Python.Validate.Syntax.Error+  , ValidationError(..)+  )+where++import Control.Lens.Prism (prism')+import Language.Python.Validate.Indentation.Error+import Language.Python.Validate.Scope.Error+import Language.Python.Validate.Syntax.Error++data ValidationError a+  = IndentationError (IndentationError a)+  | ScopeError (ScopeError a)+  | SyntaxError (SyntaxError a)+  deriving (Eq, Show)++instance AsTabError (ValidationError a) a where+  _TabError = _IndentationError._TabError++instance AsIndentationError (ValidationError a) a where+  _IndentationError =+    prism'+      IndentationError+      (\case; IndentationError a -> Just a; _ -> Nothing)++instance AsScopeError (ValidationError a) a where+  _ScopeError =+    prism'+      ScopeError+      (\case; ScopeError a -> Just a; _ -> Nothing)++instance AsSyntaxError (ValidationError a) a where+  _SyntaxError =+    prism'+      SyntaxError+      (\case; SyntaxError a -> Just a; _ -> Nothing)
+ src/Language/Python/Validate/Indentation.hs view
@@ -0,0 +1,397 @@+{-# language DataKinds, TypeOperators #-}+{-# language ScopedTypeVariables, TypeApplications #-}+{-# language LambdaCase #-}++{-|+Module      : Language.Python.Validate.Indentation+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Validate.Indentation+  ( module Data.Validation+  , module Language.Python.Validate.Indentation.Error+    -- * Main validation functions+  , Indentation, ValidateIndentation, runValidateIndentation+  , validateModuleIndentation+  , validateStatementIndentation+  , validateExprIndentation+    -- * Miscellany+    -- ** Extra types+  , NextIndent(..)+    -- ** Extra functions+  , equivalentIndentation+  , runValidateIndentation'+    -- ** Validation functions+  , validateArgsIndentation+  , validateBlockIndentation+  , validateCompoundStatementIndentation+  , validateDecoratorIndentation+  , validateExceptAsIndentation+  , validateParamsIndentation+  , validateSuiteIndentation+  )+where++import Data.Validation++import Control.Lens.Fold ((^?!), folded)+import Control.Lens.Getter ((^.))+import Control.Lens.Prism (_Right)+import Control.Lens.Review ((#))+import Control.Lens.Setter (over, mapped)+import Control.Lens.Traversal (traverseOf)+import Control.Lens.Tuple (_1, _2)+import Control.Monad.State (State, evalState, get, put)+import Data.Coerce (coerce)+import Data.Foldable (fold)+import Data.Functor.Compose (Compose(..))+import Data.List.NonEmpty (NonEmpty(..))+import Data.Type.Set+import Unsafe.Coerce (unsafeCoerce)+import Data.Validate.Monadic (ValidateM(..), liftVM0, errorVM, errorVM1)+import qualified Data.List.NonEmpty as NonEmpty++import Language.Python.Optics+import Language.Python.Optics.Validated (unvalidated)+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Module+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Whitespace+import Language.Python.Validate.Indentation.Error++data Indentation++-- | "The next line must be..."+data NextIndent+  = GreaterThan+  | EqualTo+  deriving (Eq, Show)++type ValidateIndentation e = ValidateM (NonEmpty e) (State (NextIndent, [Indent]))++runValidateIndentation :: ValidateIndentation e a -> Validation (NonEmpty e) a+runValidateIndentation = runValidateIndentation' EqualTo []++runValidateIndentation' :: NextIndent -> [Indent] -> ValidateIndentation e a -> Validation (NonEmpty e) a+runValidateIndentation' ni is =+  flip evalState (ni, is) .+  getCompose .+  unValidateM++withNextIndent+  :: (NextIndent -> [Indent] -> ValidateIndentation e a)+  -> ValidateIndentation e a+withNextIndent f =+  ValidateM . Compose $+    get >>= getCompose . unValidateM . uncurry f++checkIndent :: AsIndentationError e a => Indents a -> ValidateIndentation e (Indents a)+checkIndent i =+  withNextIndent $ \ni i' ->+  let+    a = i ^. indentsAnn+    ii = fold (i ^. indentsValue)+    ii' = fold i'+    absolute1Comparison = compare (absoluteIndentLevel 1 ii) (absoluteIndentLevel 1 ii')+    absolute8Comparison = compare (absoluteIndentLevel 8 ii) (absoluteIndentLevel 8 ii')+  in+    case ni of+      GreaterThan ->+        case (absolute1Comparison, absolute8Comparison) of+          (GT, GT) -> pure i+          (GT, _) -> errorVM $ pure (_TabError # a)+          (_, GT) -> errorVM $ pure (_TabError # a)+          (EQ, EQ) -> errorVM $ pure (_ExpectedGreaterThan # (i', i))+          (_, EQ) -> errorVM $ pure (_TabError # a)+          (EQ, _) -> errorVM $ pure (_TabError # a)+          (LT, LT) -> errorVM $ pure (_ExpectedGreaterThan # (i', i))+      EqualTo ->+        case (absolute1Comparison, absolute8Comparison) of+          (EQ, EQ) -> pure i+          (EQ, _) -> errorVM $ pure (_TabError # a)+          (_, EQ) -> errorVM $ pure (_TabError # a)+          (GT, GT) -> errorVM $ pure (_ExpectedEqualTo # (i', i))+          (_, GT) -> errorVM $ pure (_TabError # a)+          (GT, _) -> errorVM $ pure (_TabError # a)+          (LT, LT) -> errorVM $ pure (_ExpectedEqualTo # (i', i))++setNextIndent :: NextIndent -> [Indent] -> ValidateIndentation e ()+setNextIndent ni is = liftVM0 $ put (ni, is)++equivalentIndentation :: [Whitespace] -> [Whitespace] -> Bool+equivalentIndentation [] [] = True+equivalentIndentation (x:_) [] =+  case x of+    Continued _ _ -> True+    _ -> False+equivalentIndentation [] (y:_) =+  case y of+    Continued _ _ -> True+    _ -> False+equivalentIndentation (x:xs) (y:ys) =+  case (x, y) of+    (Space, Space) -> equivalentIndentation xs ys+    (Tab, Tab) -> equivalentIndentation xs ys+    (Continued _ _, Continued _ _) -> True+    _ -> False++validateBlankIndentation+  :: forall e a.+     AsIndentationError e a+  => Blank a+  -> ValidateIndentation e (Blank a)+validateBlankIndentation (Blank a ws cmt) =+  if any (\case; Continued{} -> True; _ -> False) ws+  then errorVM1 $ _EmptyContinuedLine # a+  else pure $ Blank a ws cmt++validateBlockIndentation+  :: forall e v a.+     AsIndentationError e a+  => Block v a+  -> ValidateIndentation e (Block (Nub (Indentation ': v)) a)+validateBlockIndentation (Block x b bs) =+  (\x' (b' :| bs') ->+     case b' of+       Right b'' -> Block x' b'' bs'+       _ -> error "impossible") <$>+  traverseOf (traverse._1) validateBlankIndentation x <*>+  go False (Right b) bs+  where+    is = (Right b:|bs) ^?! folded._Right.unvalidated._Indents.indentsValue++    go+      :: Bool+      -> Either+           (Blank a, Newline)+           (Statement v a)+      -> [Either (Blank a, Newline) (Statement v a)]+      -> ValidateIndentation e+         (NonEmpty+            (Either+               (Blank a, Newline)+               (Statement (Nub (Indentation ': v)) a)))+    go flag (Left e) rest =+        case rest of+          [] ->+            pure . Left <$>+            traverseOf _1 validateBlankIndentation e+          r : rs ->+            NonEmpty.cons . Left <$>+            traverseOf _1 validateBlankIndentation e <*>+            go flag r rs+    go flag (Right st) rest =+      let+        validated =+          Right <$+          (if flag then setNextIndent EqualTo is else pure ()) <*>+          validateStatementIndentation st+      in+      case rest of+        [] -> (:| []) <$> validated+        r : rs -> NonEmpty.cons <$> validated <*> go True r rs++validateSuiteIndentation+  :: AsIndentationError e a+  => Indents a+  -> Suite v a+  -> ValidateIndentation e (Suite (Nub (Indentation ': v)) a)+validateSuiteIndentation idnt (SuiteMany ann a b c d) =+  SuiteMany ann a b c <$+  setNextIndent GreaterThan (idnt ^. indentsValue) <*>+  validateBlockIndentation d+validateSuiteIndentation _ (SuiteOne ann a b) =+  SuiteOne ann a <$> validateSmallStatementIndentation b++validateExprIndentation+  :: AsIndentationError e a+  => Expr v a+  -> ValidateIndentation e (Expr (Nub (Indentation ': v)) a)+validateExprIndentation e = pure $ unsafeCoerce e++validateParamsIndentation+  :: AsIndentationError e a+  => CommaSep (Param v a)+  -> ValidateIndentation e (CommaSep (Param (Nub (Indentation ': v)) a))+validateParamsIndentation e = pure $ unsafeCoerce e++validateArgsIndentation+  :: AsIndentationError e a+  => CommaSep (Arg v a)+  -> ValidateIndentation e (CommaSep (Arg (Nub (Indentation ': v)) a))+validateArgsIndentation e = pure $ unsafeCoerce e++validateExceptAsIndentation+  :: AsIndentationError e a+  => ExceptAs v a+  -> ValidateIndentation e (ExceptAs (Nub (Indentation ': v)) a)+validateExceptAsIndentation (ExceptAs ann e f) =+  ExceptAs ann <$>+  validateExprIndentation e <*>+  pure (over (traverse._2) coerce f)++validateDecoratorIndentation+  :: AsIndentationError e a+  => Decorator v a+  -> ValidateIndentation e (Decorator (Nub (Indentation ': v)) a)+validateDecoratorIndentation (Decorator a b c d e f g) =+  (\b' -> Decorator a b' c (unsafeCoerce d) e f) <$>+  checkIndent b <*>+  traverseOf (traverse._1) validateBlankIndentation g++validateCompoundStatementIndentation+  :: forall e v a+   . AsIndentationError e a+  => CompoundStatement v a+  -> ValidateIndentation e (CompoundStatement (Nub (Indentation ': v)) a)+validateCompoundStatementIndentation (Fundef a decos idnt asyncWs ws1 name ws2 params ws3 mty s) =+  (\decos' idnt' params' ->+     Fundef a decos' idnt' asyncWs ws1 (coerce name) ws2 params' ws3 (unsafeCoerce mty)) <$>+  traverse validateDecoratorIndentation decos <*>+  checkIndent idnt <*>+  validateParamsIndentation params <*>+  validateSuiteIndentation idnt s+validateCompoundStatementIndentation (If a idnt ws1 expr s elifs body1) =+  (\idnt' -> If a idnt' ws1) <$>+  checkIndent idnt <*>+  validateExprIndentation expr <*>+  validateSuiteIndentation idnt s <*>+  traverse+    (\(idnt2, a, b, c) ->+       (,,,) <$+       setNextIndent EqualTo (idnt ^. indentsValue) <*>+       checkIndent idnt2 <*>+       pure a <*>+       validateExprIndentation b <*>+       validateSuiteIndentation idnt c)+    elifs <*>+  traverse+    (\(idnt2, a, b) ->+       (,,) <$+       setNextIndent EqualTo (idnt ^. indentsValue) <*>+       checkIndent idnt2 <*>+       pure a <*>+       validateSuiteIndentation idnt b)+    body1+validateCompoundStatementIndentation (While a idnt ws1 expr s els) =+  (\idnt' expr' -> While a idnt' ws1 expr') <$>+  checkIndent idnt <*>+  validateExprIndentation expr <*>+  validateSuiteIndentation idnt s <*>+  traverse+    (\(idnt2, a, b) ->+       (,,) <$+       setNextIndent EqualTo (idnt ^. indentsValue) <*>+       checkIndent idnt2 <*>+       pure a <*>+       validateSuiteIndentation idnt b)+    els+validateCompoundStatementIndentation (TryExcept a idnt b c d e f) =+  (\idnt' -> TryExcept a idnt' b) <$>+  checkIndent idnt <*>+  validateSuiteIndentation idnt c <*>+  traverse+    (\(a, b, c, d) ->+       (\a' -> (,,,) a' b) <$+       setNextIndent EqualTo (idnt ^. indentsValue) <*>+       checkIndent a <*>+       traverse validateExceptAsIndentation c <*>+       validateSuiteIndentation idnt d)+    d <*+  setNextIndent EqualTo (idnt ^. indentsValue) <*>+  traverse+    (\(idnt2, a, b) ->+       (\idnt2' -> (,,) idnt2' a) <$+       setNextIndent EqualTo (idnt ^. indentsValue) <*>+       checkIndent idnt2 <*>+       validateSuiteIndentation idnt b)+    e <*+  setNextIndent EqualTo (idnt ^. indentsValue) <*>+  traverse+    (\(idnt2, a, b) ->+       (\idnt2' -> (,,) idnt2' a) <$+       setNextIndent EqualTo (idnt ^. indentsValue) <*>+       checkIndent idnt2 <*>+       validateSuiteIndentation idnt b)+    f+validateCompoundStatementIndentation (TryFinally a idnt b c idnt2 d e) =+  (\idnt' c' idnt2' -> TryFinally a idnt' b c' idnt2' d) <$>+  checkIndent idnt <*>+  validateSuiteIndentation idnt c <*+  setNextIndent EqualTo (idnt ^. indentsValue) <*>+  checkIndent idnt2 <*>+  validateSuiteIndentation idnt e+validateCompoundStatementIndentation (For a idnt asyncWs b c d e h i) =+  (\idnt' c' -> For a idnt' asyncWs b c' d) <$>+  checkIndent idnt <*>+  validateExprIndentation c <*>+  traverse validateExprIndentation e <*>+  validateSuiteIndentation idnt h <*+  setNextIndent EqualTo (idnt ^. indentsValue) <*>+  traverse+    (\(idnt2, a, b) ->+       (\idnt2' -> (,,) idnt2' a) <$+       setNextIndent EqualTo (idnt ^. indentsValue) <*>+       checkIndent idnt2 <*>+       validateSuiteIndentation idnt b)+    i+validateCompoundStatementIndentation (ClassDef a decos idnt b c d e) =+  (\decos' idnt' ->+     ClassDef @(Nub (Indentation ': v)) a decos' idnt' b (coerce c) (unsafeCoerce d)) <$>+  traverse validateDecoratorIndentation decos <*>+  checkIndent idnt <*>+  validateSuiteIndentation idnt e+validateCompoundStatementIndentation (With a idnt asyncWs b c d) =+  (\idnt' -> With @(Nub (Indentation ': v)) a idnt' asyncWs b) <$>+  checkIndent idnt <*>+  traverse validateWithItemIndentation c <*>+  validateSuiteIndentation idnt d++validateWithItemIndentation+  :: AsIndentationError e a+  => WithItem v a+  -> ValidateIndentation e (WithItem (Nub (Indentation ': v)) a)+validateWithItemIndentation a = pure $ unsafeCoerce a++validateSmallStatementIndentation+  :: AsIndentationError e a+  => SmallStatement v a+  -> ValidateIndentation e (SmallStatement (Nub (Indentation ': v)) a)+validateSmallStatementIndentation (MkSmallStatement a b c d e) =+  pure $ MkSmallStatement (unsafeCoerce a) (over (mapped._2) unsafeCoerce b) c d e++validateStatementIndentation+  :: AsIndentationError e a+  => Statement v a+  -> ValidateIndentation e (Statement (Nub (Indentation ': v)) a)+validateStatementIndentation (CompoundStatement c) =+  CompoundStatement <$> validateCompoundStatementIndentation c+validateStatementIndentation (SmallStatement idnt a) =+  SmallStatement <$>+  checkIndent idnt <*>+  validateSmallStatementIndentation a++validateModuleIndentation+  :: AsIndentationError e a+  => Module v a+  -> ValidateIndentation e (Module (Nub (Indentation ': v)) a)+validateModuleIndentation m =+  case m of+    ModuleEmpty -> pure ModuleEmpty+    ModuleBlankFinal a ->+      ModuleBlankFinal <$>+      validateBlankIndentation a+    ModuleBlank a b c ->+      (\a' -> ModuleBlank a' b) <$>+      validateBlankIndentation a <*>+      validateModuleIndentation c+    ModuleStatement a b ->+     ModuleStatement <$+     setNextIndent EqualTo [] <*>+     validateStatementIndentation a <*>+     validateModuleIndentation b
+ src/Language/Python/Validate/Indentation/Error.hs view
@@ -0,0 +1,74 @@+{-# language LambdaCase #-}+{-# language TemplateHaskell, MultiParamTypeClasses, FunctionalDependencies,+  FlexibleInstances, DataKinds, KindSignatures #-}++{-|+Module      : Language.Python.Validate.Indentation.Error+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Validate.Indentation.Error+  ( IndentationError(..)+  , AsTabError(..)+  , AsIndentationError(..)+  )+where++import Control.Lens.Prism (Prism', prism')++import Language.Python.Internal.Lexer (AsTabError(..))+import Language.Python.Syntax.Whitespace+++data IndentationError a+  = IndentationTabError a+  | ExpectedGreaterThan [Indent] (Indents a)+  | ExpectedEqualTo [Indent] (Indents a)+  | EmptyContinuedLine a+  deriving (Eq, Show)++class AsTabError s a => AsIndentationError s a | s -> a where+  _IndentationError :: Prism' s (IndentationError a)++  _ExpectedGreaterThan :: Prism' s ([Indent], Indents a)+  _ExpectedGreaterThan = _IndentationError._ExpectedGreaterThan++  _ExpectedEqualTo :: Prism' s ([Indent], Indents a)+  _ExpectedEqualTo = _IndentationError._ExpectedEqualTo++  _EmptyContinuedLine :: Prism' s a+  _EmptyContinuedLine = _IndentationError._EmptyContinuedLine++instance AsTabError (IndentationError a) a where+  _TabError =+    prism'+      IndentationTabError+      (\case+          IndentationTabError a -> Just a+          _ -> Nothing)++instance AsIndentationError (IndentationError a) a where+  _IndentationError = id++  _ExpectedGreaterThan =+    prism'+      (uncurry ExpectedGreaterThan)+      (\case+          ExpectedGreaterThan a b -> Just (a, b)+          _ -> Nothing)+  _ExpectedEqualTo =+    prism'+      (uncurry ExpectedEqualTo)+      (\case+          ExpectedEqualTo a b -> Just (a, b)+          _ -> Nothing)+  _EmptyContinuedLine =+    prism'+      EmptyContinuedLine+      (\case+          EmptyContinuedLine a -> Just a+          _ -> Nothing)
+ src/Language/Python/Validate/Scope.hs view
@@ -0,0 +1,675 @@+{-# language DataKinds, TypeOperators #-}+{-# language GeneralizedNewtypeDeriving #-}+{-# language TemplateHaskell, TypeFamilies, FlexibleInstances, MultiParamTypeClasses #-}+{-# language FlexibleContexts #-}+{-# language RankNTypes #-}+{-# language LambdaCase #-}+{-# language ScopedTypeVariables, TypeApplications #-}++{-|+Module      : Language.Python.Validate.Scope+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Validate.Scope+  ( module Data.Validation+  , module Language.Python.Validate.Scope.Error+    -- * Main validation functions+  , Scope, ValidateScope, runValidateScope+  , validateModuleScope+  , validateStatementScope+  , validateExprScope+    -- * Miscellany+    -- ** Extra types+  , ScopeContext(..), scGlobalScope, scLocalScope, scImmediateScope+  , runValidateScope'+  , initialScopeContext+  , Binding(..)+    -- ** Extra functions+  , inScope+  , extendScope+  , locallyOver+  , locallyExtendOver+    -- ** Validation functions+  , validateArgScope+  , validateAssignExprScope+  , validateBlockScope+  , validateCompoundStatementScope+  , validateComprehensionScope+  , validateDecoratorScope+  , validateDictItemScope+  , validateExceptAsScope+  , validateIdentScope+  , validateListItemScope+  , validateParamScope+  , validateSetItemScope+  , validateSimpleStatementScope+  , validateSubscriptScope+  , validateSuiteScope+  , validateTupleItemScope+  )+where++import Data.Validation++import Control.Arrow ((&&&))+import Control.Applicative ((<|>))+import Control.Lens.Cons (snoc)+import Control.Lens.Fold ((^..), toListOf, folded)+import Control.Lens.Getter ((^.), to, getting, use)+import Control.Lens.Lens (Lens')+import Control.Lens.Plated (cosmos)+import Control.Lens.Prism (_Right, _Just)+import Control.Lens.Review ((#))+import Control.Lens.Setter ((%~), (.~), Setter', mapped, over)+import Control.Lens.TH (makeLenses)+import Control.Lens.Tuple (_2, _3)+import Control.Lens.Traversal (traverseOf)+import Control.Monad.State (MonadState, State, evalState, modify)+import Data.Bitraversable (bitraverse)+import Data.ByteString (ByteString)+import Data.Coerce (coerce)+import Data.Foldable (traverse_)+import Data.Functor.Compose (Compose(..))+import Data.List.NonEmpty (NonEmpty(..))+import Data.Map.Strict (Map)+import Data.String (fromString)+import Data.Type.Set (Nub)+import Data.Validate.Monadic (ValidateM(..), runValidateM, bindVM, liftVM0, errorVM1)+import Unsafe.Coerce (unsafeCoerce)++import qualified Data.List.NonEmpty as NonEmpty+import qualified Data.Map.Strict as Map++import Language.Python.Optics+import Language.Python.Optics.Validated (unvalidated)+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Module+import Language.Python.Validate.Scope.Error++data Scope++data Binding = Clean | Dirty+  deriving (Eq, Ord, Show)++data ScopeContext a+  = ScopeContext+  { _scGlobalScope :: !(Map ByteString a)+  , _scLocalScope :: !(Map ByteString a)+  , _scImmediateScope :: !(Map ByteString a)+  }+  deriving (Eq, Show)+makeLenses ''ScopeContext++initialScopeContext :: ScopeContext a+initialScopeContext = ScopeContext Map.empty Map.empty Map.empty++type ValidateScope ann e = ValidateM (NonEmpty e) (State (ScopeContext ann))++runValidateScope :: ValidateScope ann e a -> Validation (NonEmpty e) a+runValidateScope = runValidateScope' initialScopeContext++runValidateScope' :: ScopeContext ann -> ValidateScope ann e a -> Validation (NonEmpty e) a+runValidateScope' s = flip evalState s . runValidateM++extendScope+  :: Setter' (ScopeContext ann) (Map ByteString ann)+  -> [(ann, String)]+  -> ValidateScope ann e ()+extendScope l s =+  liftVM0 $ do+    gs <- use scGlobalScope+    let t = buildMap gs Map.empty+    modify (over l (t `unionL`))+  where+    buildMap gs t =+      foldr+      (\(ann, a) b ->+          let+            a' = fromString a+          in+            if Map.member a' gs+            then b+            else Map.insert a' ann b)+      t+      s++locallyOver+  :: Lens' (ScopeContext ann) b+  -> (b -> b)+  -> ValidateScope ann e a+  -> ValidateScope ann e a+locallyOver l f m =+  ValidateM . Compose $ do+    before <- use l+    modify (l %~ f)+    getCompose (unValidateM m) <* modify (l .~ before)++locallyExtendOver+  :: Lens' (ScopeContext ann) (Map ByteString ann)+  -> [(ann, String)]+  -> ValidateScope ann e a+  -> ValidateScope ann e a+locallyExtendOver l s m = locallyOver l id $ extendScope l s *> m++inScope+  :: MonadState (ScopeContext ann) m+  => String+  -> m (Maybe (Binding, ann))+inScope s = do+  gs <- use scGlobalScope+  ls <- use scLocalScope+  is <- use scImmediateScope+  let+    s' = fromString s+    inls = Map.lookup s' ls+    ings = Map.lookup s' gs+  pure $+    ((,) Clean <$> Map.lookup s' is) <|>+    (ings *> ((,) Clean <$> inls)) <|>+    ((,) Clean <$> ings) <|>+    ((,) Dirty <$> inls)++validateExceptAsScope+  :: AsScopeError e a+  => ExceptAs v a+  -> ValidateScope a e (ExceptAs (Nub (Scope ': v)) a)+validateExceptAsScope (ExceptAs ann e f) =+  ExceptAs ann <$>+  validateExprScope e <*>+  pure (over (mapped._2) coerce f)++validateSuiteScope+  :: AsScopeError e a+  => Suite v a+  -> ValidateScope a e (Suite (Nub (Scope ': v)) a)+validateSuiteScope (SuiteMany ann a b c d) = SuiteMany ann a b c <$> validateBlockScope d+validateSuiteScope (SuiteOne ann a b) =+  SuiteOne ann a <$> validateSmallStatementScope b++validateDecoratorScope+  :: AsScopeError e a+  => Decorator v a+  -> ValidateScope a e (Decorator (Nub (Scope ': v)) a)+validateDecoratorScope (Decorator a b c d e f g) =+  (\d' -> Decorator a b c d' e f g) <$>+  validateExprScope d++validateCompoundStatementScope+  :: forall e v a+   . AsScopeError e a+  => CompoundStatement v a+  -> ValidateScope a e (CompoundStatement (Nub (Scope ': v)) a)+validateCompoundStatementScope (Fundef a decos idnts asyncWs ws1 name ws2 params ws3 mty s) =+  (locallyOver scLocalScope (const Map.empty) $+   locallyOver scImmediateScope (const Map.empty) $+     (\decos' -> Fundef a decos' idnts asyncWs ws1 (coerce name) ws2) <$>+     traverse validateDecoratorScope decos <*>+     traverse validateParamScope params <*>+     pure ws3 <*>+     traverseOf (traverse._2) validateExprScope mty <*>+     locallyExtendOver+       scGlobalScope+       ((_identAnn &&& _identValue) name :+         toListOf (folded.getting paramName.to (_identAnn &&& _identValue)) params)+       (validateSuiteScope s)) <*+  extendScope scLocalScope [(_identAnn &&& _identValue) name] <*+  extendScope scImmediateScope [(_identAnn &&& _identValue) name]+validateCompoundStatementScope (If idnts a ws1 e b elifs melse) =+  use scLocalScope `bindVM` (\ls ->+  use scImmediateScope `bindVM` (\is ->+  locallyOver scGlobalScope (`unionR` unionR ls is) $+  locallyOver scImmediateScope (const Map.empty)+    (If idnts a ws1 <$>+     validateExprScope e <*>+     validateSuiteScope b <*>+     traverse+       (\(a, b, c, d) ->+          (\c' -> (,,,) a b c') <$>+          validateExprScope c <*>+          validateSuiteScope d)+       elifs <*>+     traverseOf (traverse._3) validateSuiteScope melse)))+validateCompoundStatementScope (While idnts a ws1 e b els) =+  use scLocalScope `bindVM` (\ls ->+  use scImmediateScope `bindVM` (\is ->+  locallyOver scGlobalScope (`unionR` unionR ls is) $+  locallyOver scImmediateScope (const Map.empty)+    (While idnts a ws1 <$>+     validateExprScope e <*>+     validateSuiteScope b <*>+     traverseOf (traverse._3) validateSuiteScope els)))+validateCompoundStatementScope (TryExcept idnts a b e f k l) =+  use scLocalScope `bindVM` (\ls ->+  use scImmediateScope `bindVM` (\is ->+  locallyOver scGlobalScope (`unionR` unionR ls is) $+  locallyOver scImmediateScope (const Map.empty)+    (TryExcept idnts a b <$>+     validateSuiteScope e <*>+     traverse+       (\(idnts, ws, g, h) ->+          (,,,) idnts ws <$>+          traverse validateExceptAsScope g <*>+          locallyExtendOver+            scGlobalScope+            (toListOf (folded.exceptAsName._Just._2.to (_identAnn &&& _identValue)) g)+            (validateSuiteScope h))+       f <*>+     traverseOf (traverse._3) validateSuiteScope k <*>+     traverseOf (traverse._3) validateSuiteScope l)))+validateCompoundStatementScope (TryFinally idnts a b e idnts2 f i) =+  use scLocalScope `bindVM` (\ls ->+  use scImmediateScope `bindVM` (\is ->+  locallyOver scGlobalScope (`unionR` unionR ls is) $+  locallyOver scImmediateScope (const Map.empty)+    (TryFinally idnts a b <$>+     validateSuiteScope e <*>+     pure idnts2 <*>+     pure f <*>+     validateSuiteScope i)))+validateCompoundStatementScope (For idnts a asyncWs b c d e h i) =+  use scLocalScope `bindVM` (\ls ->+  use scImmediateScope `bindVM` (\is ->+  locallyOver scGlobalScope (`unionR` unionR ls is) $+  locallyOver scImmediateScope (const Map.empty) $+    For @(Nub (Scope ': v)) idnts a asyncWs b <$>+    (unsafeCoerce c <$+     traverse+       (\s ->+          inScope (s ^. identValue) `bindVM` \res ->+          maybe (pure ()) (\_ -> errorVM1 (_BadShadowing # coerce s)) res)+       (c ^.. unvalidated.cosmos._Ident)) <*>+    pure d <*>+    traverse validateExprScope e <*>+    (let+       ls = c ^.. unvalidated.cosmos._Ident.to (_identAnn &&& _identValue)+     in+       extendScope scLocalScope ls *>+       extendScope scImmediateScope ls *>+       validateSuiteScope h) <*>+    traverseOf (traverse._3) validateSuiteScope i))+validateCompoundStatementScope (ClassDef a decos idnts b c d g) =+  (\decos' -> ClassDef @(Nub (Scope ': v)) a decos' idnts b (coerce c)) <$>+  traverse validateDecoratorScope decos <*>+  traverseOf (traverse._2.traverse.traverse) validateArgScope d <*>+  validateSuiteScope g <*+  extendScope scImmediateScope [c ^. to (_identAnn &&& _identValue)]+validateCompoundStatementScope (With a b asyncWs c d e) =+  let+    names =+      d ^..+      folded.unvalidated.to _withItemBinder.folded._2.+      assignTargets.to (_identAnn &&& _identValue)+  in+    With @(Nub (Scope ': v)) a b asyncWs c <$>+    traverse+      (\(WithItem a b c) ->+         WithItem @(Nub (Scope ': v)) a <$>+         validateExprScope b <*>+         traverseOf (traverse._2) validateAssignExprScope c)+      d <*+    extendScope scLocalScope names <*+    extendScope scImmediateScope names <*>+    validateSuiteScope e++validateSimpleStatementScope+  :: AsScopeError e a+  => SimpleStatement v a+  -> ValidateScope a e (SimpleStatement (Nub (Scope ': v)) a)+validateSimpleStatementScope (Assert a b c d) =+  Assert a b <$>+  validateExprScope c <*>+  traverseOf (traverse._2) validateExprScope d+validateSimpleStatementScope (Raise a ws f) =+  Raise a ws <$>+  traverse+    (\(b, c) ->+       (,) <$>+       validateExprScope b <*>+       traverseOf (traverse._2) validateExprScope c)+    f+validateSimpleStatementScope (Return a ws e) = Return a ws <$> traverse validateExprScope e+validateSimpleStatementScope (Expr a e) = Expr a <$> validateExprScope e+validateSimpleStatementScope (Assign a l rs) =+  let+    ls =+      (l : (snd <$> NonEmpty.init rs)) ^..+      folded.unvalidated.assignTargets.to (_identAnn &&& _identValue)+  in+  Assign a <$>+  validateAssignExprScope l <*>+  ((\a b -> case a of; [] -> b :| []; a : as -> a :| snoc as b) <$>+   traverseOf (traverse._2) validateAssignExprScope (NonEmpty.init rs) <*>+   (\(ws, b) -> (,) ws <$> validateExprScope b) (NonEmpty.last rs)) <*+  extendScope scLocalScope ls <*+  extendScope scImmediateScope ls+validateSimpleStatementScope (AugAssign a l aa r) =+  (\l' -> AugAssign a l' aa) <$>+  validateExprScope l <*>+  validateExprScope r+validateSimpleStatementScope (Global a _ _) = errorVM1 (_FoundGlobal # a)+validateSimpleStatementScope (Nonlocal a _ _) = errorVM1 (_FoundNonlocal # a)+validateSimpleStatementScope (Del a ws cs) =+  Del a ws <$+  traverse_+    (\case; Ident a -> errorVM1 (_DeletedIdent # (a ^. identAnn)); _ -> pure ())+    cs <*>+  traverse validateExprScope cs+validateSimpleStatementScope s@Pass{} = pure $ unsafeCoerce s+validateSimpleStatementScope s@Break{} = pure $ unsafeCoerce s+validateSimpleStatementScope s@Continue{} = pure $ unsafeCoerce s+validateSimpleStatementScope s@Import{} = pure $ unsafeCoerce s+validateSimpleStatementScope s@From{} = pure $ unsafeCoerce s++validateSmallStatementScope+  :: AsScopeError e a+  => SmallStatement v a+  -> ValidateScope a e (SmallStatement (Nub (Scope ': v)) a)+validateSmallStatementScope (MkSmallStatement s ss sc cmt nl) =+  (\s' ss' -> MkSmallStatement s' ss' sc cmt nl) <$>+  validateSimpleStatementScope s <*>+  traverseOf (traverse._2) validateSimpleStatementScope ss++validateStatementScope+  :: AsScopeError e a+  => Statement v a+  -> ValidateScope a e (Statement (Nub (Scope ': v)) a)+validateStatementScope (CompoundStatement c) =+  CompoundStatement <$> validateCompoundStatementScope c+validateStatementScope (SmallStatement idnts a) =+  SmallStatement idnts <$> validateSmallStatementScope a++validateIdentScope+  :: AsScopeError e a+  => Ident v a+  -> ValidateScope a e (Ident (Nub (Scope ': v)) a)+validateIdentScope i =+  inScope (_identValue i) `bindVM`+  \context ->+  case context of+    Just (Clean, _) -> pure $ coerce i+    Just (Dirty, ann)-> errorVM1 (_FoundDynamic # (ann, i ^. unvalidated))+    Nothing -> errorVM1 (_NotInScope # (i ^. unvalidated))++validateArgScope+  :: AsScopeError e a+  => Arg v a+  -> ValidateScope a e (Arg (Nub (Scope ': v)) a)+validateArgScope (PositionalArg a e) =+  PositionalArg a <$> validateExprScope e+validateArgScope (KeywordArg a ident ws2 expr) =+  KeywordArg a (coerce ident) ws2 <$> validateExprScope expr+validateArgScope (StarArg a ws e) =+  StarArg a ws <$> validateExprScope e+validateArgScope (DoubleStarArg a ws e) =+  DoubleStarArg a ws <$> validateExprScope e++validateParamScope+  :: AsScopeError e a+  => Param v a+  -> ValidateScope a e (Param (Nub (Scope ': v)) a)+validateParamScope (PositionalParam a ident mty) =+  PositionalParam a (coerce ident) <$>+  traverseOf (traverse._2) validateExprScope mty+validateParamScope (KeywordParam a ident mty ws2 expr) =+  KeywordParam a (coerce ident) <$>+  traverseOf (traverse._2) validateExprScope mty <*>+  pure ws2 <*>+  validateExprScope expr+validateParamScope (StarParam a b c d) =+  StarParam a b (coerce c) <$>+  traverseOf (traverse._2) validateExprScope d+validateParamScope (UnnamedStarParam a b) = pure $ UnnamedStarParam a b+validateParamScope (DoubleStarParam a b c d) =+  DoubleStarParam a b (coerce c) <$>+  traverseOf (traverse._2) validateExprScope d++validateBlockScope+  :: AsScopeError e a+  => Block v a+  -> ValidateScope a e (Block (Nub (Scope ': v)) a)+validateBlockScope (Block x b bs) =+  Block x <$>+  validateStatementScope b <*>+  traverseOf (traverse._Right) validateStatementScope bs++validateComprehensionScope+  :: AsScopeError e a+  => (ex v a -> ValidateScope a e (ex (Nub (Scope ': v)) a))+  -> Comprehension ex v a+  -> ValidateScope a e (Comprehension ex (Nub (Scope ': v)) a)+validateComprehensionScope f (Comprehension a b c d) =+  locallyOver scGlobalScope id $+    (\c' d' b' -> Comprehension a b' c' d') <$>+    validateCompForScope c <*>+    traverse (bitraverse validateCompForScope validateCompIfScope) d <*>+    f b+  where+    validateCompForScope+      :: AsScopeError e a+      => CompFor v a+      -> ValidateScope a e (CompFor (Nub (Scope ': v)) a)+    validateCompForScope (CompFor a b c d e) =+      (\c' -> CompFor a b c' d) <$>+      validateAssignExprScope c <*>+      validateExprScope e <*+      extendScope scGlobalScope+        (c ^.. unvalidated.assignTargets.to (_identAnn &&& _identValue))++    validateCompIfScope+      :: AsScopeError e a+      => CompIf v a+      -> ValidateScope a e (CompIf (Nub (Scope ': v)) a)+    validateCompIfScope (CompIf a b c) =+      CompIf a b <$> validateExprScope c++validateAssignExprScope+  :: AsScopeError e a+  => Expr v a+  -> ValidateScope a e (Expr (Nub (Scope ': v)) a)+validateAssignExprScope (Subscript a e1 ws1 e2 ws2) =+  (\e1' e2' -> Subscript a e1' ws1 e2' ws2) <$>+  validateAssignExprScope e1 <*>+  traverse validateSubscriptScope e2+validateAssignExprScope (List a ws1 es ws2) =+  List a ws1 <$>+  traverseOf (traverse.traverse) listItem es <*>+  pure ws2+  where+    listItem (ListItem a b) = ListItem a <$> validateAssignExprScope b+    listItem (ListUnpack a b c d) = ListUnpack a b c <$> validateAssignExprScope d+validateAssignExprScope (Deref a e ws1 r) =+  Deref a <$>+  validateExprScope e <*>+  pure ws1 <*>+  validateIdentScope r+validateAssignExprScope (Parens a ws1 e ws2) =+  Parens a ws1 <$>+  validateAssignExprScope e <*>+  pure ws2+validateAssignExprScope (Tuple a b ws d) =+  Tuple a <$>+  tupleItem b <*>+  pure ws <*>+  traverseOf (traverse.traverse) tupleItem d+  where+    tupleItem (TupleItem a b) = TupleItem a <$> validateAssignExprScope b+    tupleItem (TupleUnpack a b c d) = TupleUnpack a b c <$> validateAssignExprScope d+validateAssignExprScope e@Unit{} = pure $ unsafeCoerce e+validateAssignExprScope e@Lambda{} = pure $ unsafeCoerce e+validateAssignExprScope e@Yield{} = pure $ unsafeCoerce e+validateAssignExprScope e@YieldFrom{} = pure $ unsafeCoerce e+validateAssignExprScope e@Not{} = pure $ unsafeCoerce e+validateAssignExprScope e@ListComp{} = pure $ unsafeCoerce e+validateAssignExprScope e@Call{} = pure $ unsafeCoerce e+validateAssignExprScope e@UnOp{} = pure $ unsafeCoerce e+validateAssignExprScope e@BinOp{} = pure $ unsafeCoerce e+validateAssignExprScope e@Ident{} = pure $ unsafeCoerce e+validateAssignExprScope e@None{} = pure $ unsafeCoerce e+validateAssignExprScope e@Ellipsis{} = pure $ unsafeCoerce e+validateAssignExprScope e@Int{} = pure $ unsafeCoerce e+validateAssignExprScope e@Float{} = pure $ unsafeCoerce e+validateAssignExprScope e@Imag{} = pure $ unsafeCoerce e+validateAssignExprScope e@Bool{} = pure $ unsafeCoerce e+validateAssignExprScope e@String{} = pure $ unsafeCoerce e+validateAssignExprScope e@DictComp{} = pure $ unsafeCoerce e+validateAssignExprScope e@Dict{} = pure $ unsafeCoerce e+validateAssignExprScope e@SetComp{} = pure $ unsafeCoerce e+validateAssignExprScope e@Set{} = pure $ unsafeCoerce e+validateAssignExprScope e@Generator{} = pure $ unsafeCoerce e+validateAssignExprScope e@Await{} = pure $ unsafeCoerce e+validateAssignExprScope e@Ternary{} = pure $ unsafeCoerce e++validateDictItemScope+  :: AsScopeError e a+  => DictItem v a+  -> ValidateScope a e (DictItem (Nub (Scope ': v)) a)+validateDictItemScope (DictItem a b c d) =+  (\b' -> DictItem a b' c) <$>+  validateExprScope b <*>+  validateExprScope d+validateDictItemScope (DictUnpack a b c) =+  DictUnpack a b <$> validateExprScope c++validateSubscriptScope+  :: AsScopeError e a+  => Subscript v a+  -> ValidateScope a e (Subscript (Nub (Scope ': v)) a)+validateSubscriptScope (SubscriptExpr e) = SubscriptExpr <$> validateExprScope e+validateSubscriptScope (SubscriptSlice a b c d) =+  (\a' -> SubscriptSlice a' b) <$>+  traverse validateExprScope a <*>+  traverse validateExprScope c <*>+  traverseOf (traverse._2.traverse) validateExprScope d++validateListItemScope+  :: AsScopeError e a+  => ListItem v a+  -> ValidateScope a e (ListItem (Nub (Scope ': v)) a)+validateListItemScope (ListItem a b) = ListItem a <$> validateExprScope b+validateListItemScope (ListUnpack a b c d) = ListUnpack a b c <$> validateExprScope d++validateSetItemScope+  :: AsScopeError e a+  => SetItem v a+  -> ValidateScope a e (SetItem (Nub (Scope ': v)) a)+validateSetItemScope (SetItem a b) = SetItem a <$> validateExprScope b+validateSetItemScope (SetUnpack a b c d) = SetUnpack a b c <$> validateExprScope d++validateTupleItemScope+  :: AsScopeError e a+  => TupleItem v a+  -> ValidateScope a e (TupleItem (Nub (Scope ': v)) a)+validateTupleItemScope (TupleItem a b) = TupleItem a <$> validateExprScope b+validateTupleItemScope (TupleUnpack a b c d) = TupleUnpack a b c <$> validateExprScope d++validateExprScope+  :: AsScopeError e a+  => Expr v a+  -> ValidateScope a e (Expr (Nub (Scope ': v)) a)+validateExprScope (Lambda a b c d e) =+  Lambda a b <$>+  traverse validateParamScope c <*>+  pure d <*>+  validateExprScope e+validateExprScope (Yield a b c) =+  Yield a b <$> traverse validateExprScope c+validateExprScope (YieldFrom a b c d) =+  YieldFrom a b c <$> validateExprScope d+validateExprScope (Ternary a b c d e f) =+  (\b' d' f' -> Ternary a b' c d' e f') <$>+  validateExprScope b <*>+  validateExprScope d <*>+  validateExprScope f+validateExprScope (Subscript a b c d e) =+  (\b' d' -> Subscript a b' c d' e) <$>+  validateExprScope b <*>+  traverse validateSubscriptScope d+validateExprScope (Not a ws e) = Not a ws <$> validateExprScope e+validateExprScope (List a ws1 es ws2) =+  List a ws1 <$>+  traverseOf (traverse.traverse) validateListItemScope es <*>+  pure ws2+validateExprScope (ListComp a ws1 comp ws2) =+  ListComp a ws1 <$>+  validateComprehensionScope validateExprScope comp <*>+  pure ws2+validateExprScope (Generator a comp) =+  Generator a <$>+  validateComprehensionScope validateExprScope comp+validateExprScope (Await a ws expr) = Await a ws <$> validateExprScope expr+validateExprScope (Deref a e ws1 r) =+  Deref a <$>+  validateExprScope e <*>+  pure ws1 <*>+  validateIdentScope r+validateExprScope (Call a e ws1 as ws2) =+  Call a <$>+  validateExprScope e <*>+  pure ws1 <*>+  traverseOf (traverse.traverse) validateArgScope as <*>+  pure ws2+validateExprScope (BinOp a l op r) =+  BinOp a <$>+  validateExprScope l <*>+  pure op <*>+  validateExprScope r+validateExprScope (UnOp a op e) =+  UnOp a op <$>+  validateExprScope e+validateExprScope (Parens a ws1 e ws2) =+  Parens a ws1 <$>+  validateExprScope e <*>+  pure ws2+validateExprScope (Ident i) = Ident <$> validateIdentScope i+validateExprScope (Tuple a b ws d) =+  Tuple a <$>+  validateTupleItemScope b <*>+  pure ws <*>+  traverseOf (traverse.traverse) validateTupleItemScope d+validateExprScope e@None{} = pure $ unsafeCoerce e+validateExprScope e@Ellipsis{} = pure $ unsafeCoerce e+validateExprScope e@Int{} = pure $ unsafeCoerce e+validateExprScope e@Float{} = pure $ unsafeCoerce e+validateExprScope e@Imag{} = pure $ unsafeCoerce e+validateExprScope e@Bool{} = pure $ unsafeCoerce e+validateExprScope e@String{} = pure $ unsafeCoerce e+validateExprScope e@Unit{} = pure $ unsafeCoerce e+validateExprScope (DictComp a ws1 comp ws2) =+  DictComp a ws1 <$>+  validateComprehensionScope validateDictItemScope comp <*>+  pure ws2+validateExprScope (Dict a b c d) =+  (\c' -> Dict a b c' d) <$> traverseOf (traverse.traverse) validateDictItemScope c+validateExprScope (SetComp a ws1 comp ws2) =+  SetComp a ws1 <$>+  validateComprehensionScope validateSetItemScope comp <*>+  pure ws2+validateExprScope (Set a b c d) =+  (\c' -> Set a b c' d) <$> traverse validateSetItemScope c++validateModuleScope+  :: AsScopeError e a+  => Module v a+  -> ValidateScope a e (Module (Nub (Scope ': v)) a)+validateModuleScope m =+  case m of+    ModuleEmpty -> pure ModuleEmpty+    ModuleBlankFinal a -> pure $ ModuleBlankFinal a+    ModuleBlank a b c -> ModuleBlank a b <$> validateModuleScope c+    ModuleStatement a b ->+     ModuleStatement <$>+     validateStatementScope a <*>+     validateModuleScope b++unionL :: Ord k => Map k v -> Map k v -> Map k v+unionL = Map.unionWith const++unionR :: Ord k => Map k v -> Map k v -> Map k v+unionR = Map.unionWith (const id)
+ src/Language/Python/Validate/Scope/Error.hs view
@@ -0,0 +1,68 @@+{-# language DataKinds, KindSignatures #-}+{-# language TemplateHaskell #-}+{-# language MultiParamTypeClasses, FunctionalDependencies, FlexibleInstances #-}++{-|+Module      : Language.Python.Validate.Scope.Error+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Validate.Scope.Error where++import Control.Lens.TH+import Language.Python.Syntax.Ident++data ScopeError a+  -- |+  -- Using @nonlocal@ to modify function scopes makes scope checking intractible+  = FoundNonlocal a+  -- |+  -- Using @global@ to add identifiers to the global scope makes scope checking+  -- intractible+  | FoundGlobal a+  -- |+  -- Using @del@ to remove identifiers from scope makes scope checking intractible+  | DeletedIdent a+  -- |+  -- Variable assignments deep in control flow can modify the scope outside+  -- the control flow. For example:+  --+  -- @+  -- if a:+  --     x = 0+  -- else:+  --     pass+  --+  -- print(x)+  -- @+  --+  -- @x@ will be in scope if the @True@ branch was entered, but not if the @False@+  -- branch was entered. This kind of behaviour makes scope checking intractible, so+  -- programs like this are considered scope errors.+  | FoundDynamic a (Ident '[] a)+  -- | An identifier is not in scope+  | NotInScope (Ident '[] a)+  -- |+  -- For loops don't execute in a fresh scope, so if the 'counter' of the loop+  -- shadows a variable, then that variable will be mutated.+  --+  -- e.g.+  --+  -- @+  -- x = 0+  -- for x in 1, 2, 3:+  --    pass+  -- print(x)+  -- @+  --+  -- outputs @3@+  --+  -- This error occurs when we spot this pattern.+  | BadShadowing (Ident '[] a)+  deriving (Eq, Show)++makeClassyPrisms ''ScopeError
+ src/Language/Python/Validate/Syntax.hs view
@@ -0,0 +1,1330 @@+{-# language DataKinds #-}+{-# language GeneralizedNewtypeDeriving #-}+{-# language FlexibleContexts #-}+{-# language PolyKinds #-}+{-# language TypeOperators #-}+{-# language TypeSynonymInstances, FlexibleInstances #-}+{-# language TemplateHaskell, TypeFamilies, MultiParamTypeClasses #-}+{-# language RankNTypes #-}+{-# language LambdaCase #-}+{-# language ScopedTypeVariables #-}++{-|+Module      : Language.Python.Validate.Syntax+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Validate.Syntax+  ( module Data.Validation+  , module Language.Python.Validate.Syntax.Error+    -- * Main validation functions+  , Syntax, ValidateSyntax, runValidateSyntax+  , validateModuleSyntax+  , validateStatementSyntax+  , validateExprSyntax+    -- * Miscellany+    -- ** Extra types+  , SyntaxContext(..), FunctionInfo(..), inLoop, inFunction, inGenerator, inParens+  , runValidateSyntax'+  , initialSyntaxContext+    -- ** Extra functions+  , reservedWords+  , canAssignTo+  , deleteBy'+  , deleteFirstsBy'+  , localNonlocals+    -- ** Validation functions+  , validateArgsSyntax+  , validateBlockSyntax+  , validateCompoundStatementSyntax+  , validateComprehensionSyntax+  , validateDecoratorSyntax+  , validateDictItemSyntax+  , validateExceptAsSyntax+  , validateIdentSyntax+  , validateImportAsSyntax+  , validateImportTargetsSyntax+  , validateListItemSyntax+  , validateParamsSyntax+  , validateSetItemSyntax+  , validateSimpleStatementSyntax+  , validateStringLiteralSyntax+  , validateSubscriptSyntax+  , validateSuiteSyntax+  , validateTupleItemSyntax+  , validateWhitespace+  )+where++import Data.Validation++import Control.Applicative ((<|>), liftA2)+import Control.Lens.Cons (snoc, _init)+import Control.Lens.Fold+  ((^..), (^?), (^?!), folded, allOf, toListOf, anyOf, lengthOf, has)+import Control.Lens.Getter ((^.), getting, view)+import Control.Lens.Prism (_Right, _Just)+import Control.Lens.Review ((#))+import Control.Lens.Setter ((.~), (%~))+import Control.Lens.TH (makeLenses)+import Control.Lens.Tuple (_1, _2, _3)+import Control.Lens.Traversal (traverseOf)+import Control.Monad (when)+import Control.Monad.State (State, put, modify, get, evalState)+import Control.Monad.Reader (ReaderT, local, ask, runReaderT)+import Data.Char (isAscii, ord)+import Data.Coerce (coerce)+import Data.Foldable (toList, traverse_)+import Data.Bitraversable (bitraverse)+import Data.Functor.Compose (Compose(..))+import Data.List (intersect, union)+import Data.List.NonEmpty (NonEmpty(..), (<|))+import Data.Maybe (isJust, isNothing, fromMaybe)+import Data.Semigroup (Semigroup(..))+import Data.Type.Set (Nub, Member)+import Data.Validate.Monadic (ValidateM(..), bindVM, liftVM0, liftVM1, errorVM, errorVM1)+import Unsafe.Coerce (unsafeCoerce)++import qualified Data.List.NonEmpty as NonEmpty++import Language.Python.Optics+import Language.Python.Optics.Validated (unvalidated)+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Ident+import Language.Python.Syntax.Import+import Language.Python.Syntax.Module+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace+import Language.Python.Validate.Indentation+import Language.Python.Validate.Syntax.Error++deleteBy' :: (a -> b -> Bool) -> a -> [b] -> [b]+deleteBy' _ _ [] = []+deleteBy' eq a (b:bs) = if a `eq` b then bs else b : deleteBy' eq a bs++deleteFirstsBy' :: (a -> b -> Bool) -> [a] -> [b] -> [a]+deleteFirstsBy' eq = foldl (flip (deleteBy' (flip eq)))++reservedWords :: [String]+reservedWords =+  [ "False"+  , "class"+  , "finally"+  , "is"+  , "return"+  , "None"+  , "continue"+  , "for"+  , "lambda"+  , "try"+  , "True"+  , "def"+  , "from"+  , "nonlocal"+  , "while"+  , "and"+  , "del"+  , "global"+  , "not"+  , "with"+  , "as"+  , "elif"+  , "if"+  , "or"+  , "yield"+  , "assert"+  , "else"+  , "import"+  , "pass"+  , "break"+  , "except"+  , "in"+  , "raise"+  ]++data Syntax++data FunctionInfo+  = FunctionInfo+  { _functionParams :: [String]+  , _asyncFunction :: Bool+  }+makeLenses ''FunctionInfo++data SyntaxContext+  = SyntaxContext+  { _inLoop :: Bool+  , _inFinally :: Bool+  , _inFunction :: Maybe FunctionInfo+  , _inGenerator :: Bool+  , _inClass :: Bool+  , _inParens :: Bool+  }+makeLenses ''SyntaxContext++type ValidateSyntax e = ValidateM (NonEmpty e) (ReaderT SyntaxContext (State [String]))++runValidateSyntax :: ValidateSyntax e a -> Validation (NonEmpty e) a+runValidateSyntax = runValidateSyntax' initialSyntaxContext []++runValidateSyntax' :: SyntaxContext -> [String] -> ValidateSyntax e a -> Validation (NonEmpty e) a+runValidateSyntax' ctxt nlscope =+  flip evalState nlscope .+  flip runReaderT ctxt . getCompose .+  unValidateM++localNonlocals :: ([String] -> [String]) -> ValidateSyntax e a -> ValidateSyntax e a+localNonlocals f v =+  ValidateM . Compose $ do+    before <- get+    modify f+    res <- getCompose $ unValidateM v+    put before+    pure res++initialSyntaxContext :: SyntaxContext+initialSyntaxContext =+  SyntaxContext+  { _inLoop = False+  , _inFinally = False+  , _inFunction = Nothing+  , _inGenerator = False+  , _inClass = False+  , _inParens = False+  }++validateIdentSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Ident v a+  -> ValidateSyntax e (Ident (Nub (Syntax ': v)) a)+validateIdentSyntax (MkIdent a name ws)+  | not (all isAscii name) = errorVM1 (_BadCharacter # (a, name))+  | null name = errorVM1 (_EmptyIdentifier # a)+  | otherwise =+      bindVM (view inFunction) $ \fi ->+        let+          reserved =+            reservedWords <>+            if fromMaybe False (fi ^? _Just.asyncFunction)+            then ["async", "await"]+            else []+        in+          if (name `elem` reserved)+            then errorVM1 (_IdentifierReservedWord # (a, name))+            else pure $ MkIdent a name ws++validateWhitespace+  :: (AsSyntaxError e a, Foldable f)+  => a+  -> f Whitespace+  -> ValidateSyntax e (f Whitespace)+validateWhitespace ann ws =+  ask `bindVM` \ctxt ->+  if _inParens ctxt+  then pure ws+  else if+    any+      (\case+          Newline{} -> True+          Comment{} -> False+          Continued{} -> False+          Tab -> False+          Space -> False)+      ws+  then errorVM1 (_UnexpectedNewline # ann)+  else if+    any+      (\case+          Newline{} -> False+          Comment{} -> True+          Continued{} -> False+          Tab -> False+          Space -> False)+      ws+  then errorVM1 (_UnexpectedComment # ann)+  else pure ws++validateAt+  :: (AsSyntaxError e a)+  => a+  -> At+  -> ValidateSyntax e At+validateAt a (MkAt ws) = MkAt <$> validateWhitespace a ws++validateComma+  :: (AsSyntaxError e a)+  => a+  -> Comma+  -> ValidateSyntax e Comma+validateComma a (MkComma ws) = MkComma <$> validateWhitespace a ws++validateColon+  :: (AsSyntaxError e a)+  => a+  -> Colon+  -> ValidateSyntax e Colon+validateColon a (MkColon ws) = MkColon <$> validateWhitespace a ws++validateSemicolon+  :: AsSyntaxError e a+  => Semicolon a+  -> ValidateSyntax e (Semicolon a)+validateSemicolon (MkSemicolon a ws) = MkSemicolon a <$> validateWhitespace a ws++validateEquals+  :: AsSyntaxError e a+  => a+  -> Equals+  -> ValidateSyntax e Equals+validateEquals a (MkEquals ws) = MkEquals <$> validateWhitespace a ws++validateAssignmentSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => a+  -> Expr v a+  -> ValidateSyntax e (Expr (Nub (Syntax ': v)) a)+validateAssignmentSyntax a ex =+  (if+     lengthOf (getting $ _Tuple.tupleItems._TupleUnpack) ex > 1 ||+     lengthOf (getting $ _List.listItems._ListUnpack) ex > 1+   then errorVM1 $ _ManyStarredTargets # a+   else pure ()) *>+  (if canAssignTo ex+   then validateExprSyntax ex+   else errorVM1 $ _CannotAssignTo # (a, ex ^. unvalidated))++validateCompForSyntax+  :: ( AsSyntaxError e a+    , Member Indentation v+    )+  => CompFor v a+  -> ValidateSyntax e (CompFor (Nub (Syntax ': v)) a)+validateCompForSyntax (CompFor a b c d e) =+  (\c' -> CompFor a b c' d) <$>+  liftVM1 (local $ inGenerator .~ True) (validateAssignmentSyntax a c) <*>+  validateExprSyntax e++validateCompIfSyntax+  :: ( AsSyntaxError e a+    , Member Indentation v+    )+  => CompIf v a+  -> ValidateSyntax e (CompIf (Nub (Syntax ': v)) a)+validateCompIfSyntax (CompIf a b c) =+  CompIf a b <$> liftVM1 (local $ inGenerator .~ True) (validateExprSyntax c)++validateComprehensionSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => (ex v a -> ValidateSyntax e (ex (Nub (Syntax ': v)) a))+  -> Comprehension ex v a+  -> ValidateSyntax e (Comprehension ex (Nub (Syntax ': v)) a)+validateComprehensionSyntax f (Comprehension a b c d) =+  Comprehension a <$>+  liftVM1 (local $ inGenerator .~ True) (f b) <*>+  validateCompForSyntax c <*>+  liftVM1+    (local $ inGenerator .~ True)+    (traverse+      (bitraverse validateCompForSyntax validateCompIfSyntax)+      d)++validateStringPyChar+  :: ( AsSyntaxError e a+     )+  => a+  -> PyChar+  -> ValidateSyntax e PyChar+validateStringPyChar a (Char_lit '\0') =+  errorVM1 $ _NullByte # a+validateStringPyChar _ a = pure a++validateBytesPyChar+  :: ( AsSyntaxError e a+     )+  => a+  -> PyChar+  -> ValidateSyntax e PyChar+validateBytesPyChar a (Char_lit '\0') =+  errorVM1 $ _NullByte # a+validateBytesPyChar a (Char_lit c) | ord c >= 128 =+  errorVM1 $ _NonAsciiInBytes # (a, c)+validateBytesPyChar _ a = pure a++validateStringLiteralSyntax+  :: AsSyntaxError e a+  => StringLiteral a+  -> ValidateSyntax e (StringLiteral a)+validateStringLiteralSyntax (StringLiteral a b c d e f) =+  StringLiteral a b c d <$>+  traverse (validateStringPyChar a) e <*>+  validateWhitespace a f+validateStringLiteralSyntax (BytesLiteral a b c d e f) =+  BytesLiteral a b c d <$>+  traverse (validateBytesPyChar a) e <*>+  validateWhitespace a f+validateStringLiteralSyntax (RawStringLiteral a b c d e f) =+  RawStringLiteral a b c d e <$>+  validateWhitespace a f+validateStringLiteralSyntax (RawBytesLiteral a b c d e f) =+  RawBytesLiteral a b c d e <$>+  validateWhitespace a f++validateDictItemSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => DictItem v a+  -> ValidateSyntax e (DictItem (Nub (Syntax ': v)) a)+validateDictItemSyntax (DictItem a b c d) =+  (\b' -> DictItem a b' c) <$>+  validateExprSyntax b <*>+  validateExprSyntax d+validateDictItemSyntax (DictUnpack a b c) =+  DictUnpack a <$>+  validateWhitespace a b <*>+  validateExprSyntax c++validateSubscriptSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Subscript v a+  -> ValidateSyntax e (Subscript (Nub (Syntax ': v)) a)+validateSubscriptSyntax (SubscriptExpr e) = SubscriptExpr <$> validateExprSyntax e+validateSubscriptSyntax (SubscriptSlice a b c d) =+  (\a' -> SubscriptSlice a' b) <$>+  traverse validateExprSyntax a <*>+  traverse validateExprSyntax c <*>+  traverseOf (traverse._2.traverse) validateExprSyntax d++validateListItemSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => ListItem v a+  -> ValidateSyntax e (ListItem (Nub (Syntax ': v)) a)+validateListItemSyntax (ListItem a b) = ListItem a <$> validateExprSyntax b+validateListItemSyntax (ListUnpack a b c d) =+  ListUnpack a <$>+  traverseOf (traverse._2) (validateWhitespace a) b <*>+  validateWhitespace a c <*>+  validateExprSyntax d++validateSetItemSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => SetItem v a+  -> ValidateSyntax e (SetItem (Nub (Syntax ': v)) a)+validateSetItemSyntax (SetItem a b) = SetItem a <$> validateExprSyntax b+validateSetItemSyntax (SetUnpack a b c d) =+  SetUnpack a <$>+  traverseOf (traverse._2) (validateWhitespace a) b <*>+  validateWhitespace a c <*>+  validateExprSyntax d++validateTupleItemSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => TupleItem v a+  -> ValidateSyntax e (TupleItem (Nub (Syntax ': v)) a)+validateTupleItemSyntax (TupleItem a b) = TupleItem a <$> validateExprSyntax b+validateTupleItemSyntax (TupleUnpack a b c d) =+  TupleUnpack a <$>+  traverseOf (traverse._2) (validateWhitespace a) b <*>+  validateWhitespace a c <*>+  validateExprSyntax d++validateExprSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Expr v a+  -> ValidateSyntax e (Expr (Nub (Syntax ': v)) a)+validateExprSyntax (Unit a b c) =+  Unit a <$>+  liftVM1 (local $ inParens .~ True) (validateWhitespace a b) <*>+  validateWhitespace a c+validateExprSyntax (Lambda a b c d e) =+  let+    paramIdents = c ^.. folded.unvalidated.paramName.identValue+  in+    Lambda a <$>+    validateWhitespace a b <*>+    validateParamsSyntax True c <*>+    validateColon a d <*>+    liftVM1+      (local $+       \ctxt ->+          ctxt+          { _inLoop = False+          , _inFunction =+              fmap+                ((functionParams %~ (`union` paramIdents)) . (asyncFunction .~ False))+                (_inFunction ctxt) <|>+              Just (FunctionInfo paramIdents False)+          })+      (validateExprSyntax e)+validateExprSyntax (Yield a b c) =+  Yield a <$>+  validateWhitespace a b <*+  (ask `bindVM` \ctxt ->+      case _inFunction ctxt of+        Nothing+          | _inGenerator ctxt -> pure ()+          | otherwise -> errorVM1 (_YieldOutsideGenerator # a)+        Just info ->+          if info^.asyncFunction+          then errorVM1 $ _YieldInsideCoroutine # a+          else pure ()) <*>+  traverse validateExprSyntax c+validateExprSyntax (YieldFrom a b c d) =+  YieldFrom a <$>+  validateWhitespace a b <*>+  validateWhitespace a c <*+  (ask `bindVM` \ctxt ->+      case _inFunction ctxt of+        Nothing+          | _inGenerator ctxt -> pure ()+          | otherwise -> errorVM1 (_YieldOutsideGenerator # a)+        Just fi ->+          if fi ^. asyncFunction+          then errorVM1 (_YieldFromInsideCoroutine # a)+          else pure ()) <*>+  validateExprSyntax d+validateExprSyntax (Ternary a b c d e f) =+  (\b' d' f' -> Ternary a b' c d' e f') <$>+  validateExprSyntax b <*>+  validateExprSyntax d <*>+  validateExprSyntax f+validateExprSyntax (Subscript a b c d e) =+  (\b' d' -> Subscript a b' c d' e) <$>+  validateExprSyntax b <*>+  traverse validateSubscriptSyntax d+validateExprSyntax (Not a ws e) =+  Not a <$>+  validateWhitespace a ws <*>+  validateExprSyntax e+validateExprSyntax (Parens a ws1 e ws2) =+  Parens a ws1 <$>+  liftVM1 (local $ inParens .~ True) (validateExprSyntax e) <*>+  validateWhitespace a ws2+validateExprSyntax (Bool a b ws) = pure $ Bool a b ws+validateExprSyntax (UnOp a op expr) =+  UnOp a op <$> validateExprSyntax expr+validateExprSyntax (String a strLits) =+  if+    all+      (\case+          StringLiteral{} -> True+          RawStringLiteral{} -> True+          _ -> False)+      strLits+      ||+    all+      (\case+          BytesLiteral{} -> True+          RawBytesLiteral{} -> True+          _ -> False)+      strLits+  then+    String a <$> traverse validateStringLiteralSyntax strLits+  else+    errorVM1 (_Can'tJoinStringAndBytes # a)+validateExprSyntax (Int a n ws) = pure $ Int a n ws+validateExprSyntax (Float a n ws) = pure $ Float a n ws+validateExprSyntax (Imag a n ws) = pure $ Imag a n ws+validateExprSyntax (Ident name) = Ident <$> validateIdentSyntax name+validateExprSyntax (List a ws1 exprs ws2) =+  List a ws1 <$>+  liftVM1+    (local $ inParens .~ True)+    (traverseOf (traverse.traverse) validateListItemSyntax exprs) <*>+  validateWhitespace a ws2+validateExprSyntax (ListComp a ws1 comp ws2) =+  liftVM1+    (local $ inParens .~ True)+    (ListComp a ws1 <$>+     validateComprehensionSyntax validateExprSyntax comp) <*>+  validateWhitespace a ws2+validateExprSyntax (Generator a comp) =+  Generator a <$> validateComprehensionSyntax validateExprSyntax comp+validateExprSyntax (Await a ws expr) =+  bindVM ask $ \ctxt ->+  Await a <$>+  validateWhitespace a ws <*+  (if not $ fromMaybe False (ctxt ^? inFunction._Just.asyncFunction)+   then errorVM1 $ _AwaitOutsideCoroutine # a+   else pure () *>+   if ctxt^.inGenerator+   then errorVM1 $ _AwaitInsideComprehension # a+   else pure ()) <*>+  validateExprSyntax expr+validateExprSyntax (Deref a expr ws1 name) =+  Deref a <$>+  validateExprSyntax expr <*>+  validateWhitespace a ws1 <*>+  validateIdentSyntax name+validateExprSyntax (Call a expr ws args ws2) =+  Call a <$>+  validateExprSyntax expr <*>+  liftVM1 (local $ inParens .~ True) (validateWhitespace a ws) <*>+  liftVM1 (local $ inParens .~ True) (traverse validateArgsSyntax args) <*>+  validateWhitespace a ws2+validateExprSyntax (None a ws) = None a <$> validateWhitespace a ws+validateExprSyntax (Ellipsis a ws) = Ellipsis a <$> validateWhitespace a ws+validateExprSyntax (BinOp a e1 op e2) =+  BinOp a <$>+  validateExprSyntax e1 <*>+  pure op <*>+  validateExprSyntax e2+validateExprSyntax (Tuple a b comma d) =+  Tuple a <$>+  validateTupleItemSyntax b <*>+  validateComma a comma <*>+  traverseOf (traverse.traverse) validateTupleItemSyntax d+validateExprSyntax (DictComp a ws1 comp ws2) =+  liftVM1+    (local $ inParens .~ True)+    (DictComp a ws1 <$>+     validateComprehensionSyntax dictItem comp) <*>+  validateWhitespace a ws2+  where+    dictItem (DictUnpack a _ _) = errorVM1 (_InvalidDictUnpacking # a)+    dictItem a = validateDictItemSyntax a+validateExprSyntax (Dict a b c d) =+  Dict a b <$>+  liftVM1+    (local $ inParens .~ True)+    (traverseOf (traverse.traverse) validateDictItemSyntax c) <*>+  validateWhitespace a d+validateExprSyntax (SetComp a ws1 comp ws2) =+  liftVM1+    (local $ inParens .~ True)+    (SetComp a ws1 <$>+     validateComprehensionSyntax setItem comp) <*>+  validateWhitespace a ws2+  where+    setItem (SetUnpack a _ _ _) = errorVM1 (_InvalidSetUnpacking # a)+    setItem a = validateSetItemSyntax a+validateExprSyntax (Set a b c d) =+  Set a b <$>+  liftVM1+    (local $ inParens .~ True)+    (traverse validateSetItemSyntax c) <*>+  validateWhitespace a d++validateBlockSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Block v a+  -> ValidateSyntax e (Block (Nub (Syntax ': v)) a)+validateBlockSyntax (Block x b bs) =+  Block x <$>+  validateStatementSyntax b <*>+  traverseOf (traverse._Right) validateStatementSyntax bs++validateSuiteSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Suite v a+  -> ValidateSyntax e (Suite (Nub (Syntax ': v)) a)+validateSuiteSyntax (SuiteMany a b c d e) =+  (\b' -> SuiteMany a b' c d) <$>+  validateColon a b <*>+  validateBlockSyntax e+validateSuiteSyntax (SuiteOne a b c) =+  SuiteOne a <$>+  validateColon a b <*>+  validateSmallStatementSyntax c++validateDecoratorSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Decorator v a+  -> ValidateSyntax e (Decorator (Nub (Syntax ': v)) a)+validateDecoratorSyntax (Decorator a b c d e f g) =+  (\c' d' -> Decorator a b c' d' e f) <$>+  validateAt a c <*>+  isDecoratorValue d <*>+  traverseOf (traverse._1) validateBlankSyntax g+  where+    someDerefs Ident{} = True+    someDerefs (Deref _ a _ _) = someDerefs a+    someDerefs _ = False++    isDecoratorValue e@(Call _ a _ _ _) | someDerefs a = pure $ unsafeCoerce e+    isDecoratorValue e | someDerefs e = pure $ unsafeCoerce e+    isDecoratorValue _ = errorVM1 (_MalformedDecorator # a)++validateBlankSyntax :: AsSyntaxError e a => Blank a -> ValidateSyntax e (Blank a)+validateBlankSyntax (Blank a ws cmt) =+  (\ws' -> Blank a ws' cmt) <$>+  validateWhitespace a ws++validateCompoundStatementSyntax+  :: forall e v a+   . ( AsSyntaxError e a+     , Member Indentation v+     )+  => CompoundStatement v a+  -> ValidateSyntax e (CompoundStatement (Nub (Syntax ': v)) a)+validateCompoundStatementSyntax (Fundef a decos idnts asyncWs ws1 name ws2 params ws3 mty body) =+  let+    paramIdents = params ^.. folded.unvalidated.paramName.identValue+  in+    (\decos' -> Fundef a decos' idnts) <$>+    traverse validateDecoratorSyntax decos <*>+    traverse (validateWhitespace a) asyncWs <*>+    validateWhitespace a ws1 <*>+    validateIdentSyntax name <*>+    pure ws2 <*>+    liftVM1 (local $ inParens .~ True) (validateParamsSyntax False params) <*>+    pure ws3 <*>+    traverse (bitraverse (validateWhitespace a) validateExprSyntax) mty <*>+    localNonlocals id+      (liftVM1+         (local $+          \ctxt ->+            ctxt+            { _inLoop = False+            , _inFunction =+                fmap+                  ((functionParams %~ (`union` paramIdents)) .+                   (asyncFunction %~ (|| isJust asyncWs)))+                  (_inFunction ctxt) <|>+                Just (FunctionInfo paramIdents $ isJust asyncWs)+            })+         (validateSuiteSyntax body))+validateCompoundStatementSyntax (If a idnts ws1 expr body elifs body') =+  If a idnts <$>+  validateWhitespace a ws1 <*>+  validateExprSyntax expr <*>+  validateSuiteSyntax body <*>+  traverse+    (\(a, b, c, d) ->+       (\c' -> (,,,) a b c') <$>+       validateExprSyntax c <*>+       validateSuiteSyntax d)+    elifs <*>+  traverseOf (traverse._3) validateSuiteSyntax body'+validateCompoundStatementSyntax (While a idnts ws1 expr body els) =+  While a idnts <$>+  validateWhitespace a ws1 <*>+  validateExprSyntax expr <*>+  liftVM1 (local $ (inFinally .~ False) . (inLoop .~ True)) (validateSuiteSyntax body) <*>+  traverseOf (traverse._3) validateSuiteSyntax els+validateCompoundStatementSyntax (TryExcept a idnts b e f k l) =+  TryExcept a idnts <$>+  validateWhitespace a b <*>+  validateSuiteSyntax e <*>+  traverse+    (\(idnts, f, g, j) ->+       (,,,) idnts <$>+       validateWhitespace a f <*>+       traverse validateExceptAsSyntax g <*>+       validateSuiteSyntax j)+    f <*+  (if anyOf (_init.folded._3) isNothing $ NonEmpty.toList f+   then errorVM1 $ _DefaultExceptMustBeLast # a+   else pure ()) <*>+  traverse+    (\(idnts, x, w) ->+       (,,) idnts <$>+       validateWhitespace a x <*>+       validateSuiteSyntax w)+    k <*>+  traverse+    (\(idnts, x, w) ->+       (,,) idnts <$>+       validateWhitespace a x <*>+       liftVM1 (local $ inFinally .~ True) (validateSuiteSyntax w))+    l+validateCompoundStatementSyntax (TryFinally a idnts b e idnts2 f i) =+  TryFinally a idnts <$>+  validateWhitespace a b <*>+  validateSuiteSyntax e <*> pure idnts2 <*>+  validateWhitespace a f <*>+  liftVM1 (local $ inFinally .~ True) (validateSuiteSyntax i)+validateCompoundStatementSyntax (ClassDef a decos idnts b c d g) =+  liftVM1 (local $ inLoop .~ False) $+  (\decos' -> ClassDef a decos' idnts) <$>+  traverse validateDecoratorSyntax decos <*>+  validateWhitespace a b <*>+  validateIdentSyntax c <*>+  traverse+    (\(x, y, z) ->+       (,,) <$>+       validateWhitespace a x <*>+       traverse+         (liftVM1 (local $ inParens .~ True) . validateArgsSyntax)+         y <*>+       validateWhitespace a z)+    d <*>+  liftVM1+    (local $ (inClass .~ True) . (inFunction .~ Nothing))+    (validateSuiteSyntax g)+validateCompoundStatementSyntax (For a idnts asyncWs b c d e h i) =+  bindVM ask $ \ctxt ->+  For a idnts <$+  (if isJust asyncWs && not (fromMaybe False $ ctxt ^? inFunction._Just.asyncFunction)+   then errorVM1 (_AsyncForOutsideCoroutine # a)+   else pure ()) <*>+  traverse (validateWhitespace a) asyncWs <*>+  validateWhitespace a b <*>+  validateAssignmentSyntax a c <*>+  validateWhitespace a d <*>+  traverse validateExprSyntax e <*>+  liftVM1+    (local $ (inFinally .~ False) . (inLoop .~ True))+    (validateSuiteSyntax h) <*>+  traverse+    (\(idnts, x, w) ->+       (,,) idnts <$>+       validateWhitespace a x <*>+       validateSuiteSyntax w)+    i+validateCompoundStatementSyntax (With a b asyncWs c d e) =+  bindVM ask $ \ctxt ->+  With a b <$+  (if isJust asyncWs && not (fromMaybe False $ ctxt ^? inFunction._Just.asyncFunction)+   then errorVM1 (_AsyncWithOutsideCoroutine # a)+   else pure ()) <*>+  traverse (validateWhitespace a) asyncWs <*>+  validateWhitespace a c <*>+  traverse+    (\(WithItem a b c) ->+        WithItem a <$>+        validateExprSyntax b <*>+        traverse+          (\(ws, b) -> (,) <$> validateWhitespace a ws <*> validateAssignmentSyntax a b)+          c)+    d <*>+  validateSuiteSyntax e++validateExceptAsSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => ExceptAs v a+  -> ValidateSyntax e (ExceptAs (Nub (Syntax ': v)) a)+validateExceptAsSyntax (ExceptAs ann e f) =+  ExceptAs ann <$>+  validateExprSyntax e <*>+  traverse (\(a, b) -> (,) <$> validateWhitespace ann a <*> validateIdentSyntax b) f++validateImportAsSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => (t a -> ValidateSyntax e (t' a))+  -> ImportAs t v a+  -> ValidateSyntax e (ImportAs t' (Nub (Syntax ': v)) a)+validateImportAsSyntax v (ImportAs x a b) =+  ImportAs x <$>+  v a <*>+  traverse+    (\(c, d) ->+       (,) <$>+       (c <$ validateWhitespace x (NonEmpty.toList c)) <*>+       validateIdentSyntax d)+    b++validateImportTargetsSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => ImportTargets v a+  -> ValidateSyntax e (ImportTargets (Nub (Syntax ': v)) a)+validateImportTargetsSyntax (ImportAll a ws) =+  bindVM ask $ \ctxt ->+  if ctxt ^. inClass || has (inFunction._Just) ctxt+    then errorVM1 $ _WildcardImportInDefinition # a+    else ImportAll a <$> validateWhitespace a ws+validateImportTargetsSyntax (ImportSome a cs) =+  ImportSome a <$> traverse (validateImportAsSyntax validateIdentSyntax) cs+validateImportTargetsSyntax (ImportSomeParens a ws1 cs ws2) =+  liftVM1+    (local $ inParens .~ True)+    (ImportSomeParens a <$>+     validateWhitespace a ws1 <*>+     traverse (validateImportAsSyntax validateIdentSyntax) cs) <*>+  validateWhitespace a ws2++validateSimpleStatementSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => SimpleStatement v a+  -> ValidateSyntax e (SimpleStatement (Nub (Syntax ': v)) a)+validateSimpleStatementSyntax (Assert a b c d) =+  Assert a <$>+  validateWhitespace a b <*>+  validateExprSyntax c <*>+  traverseOf (traverse._2) validateExprSyntax d+validateSimpleStatementSyntax (Raise a ws f) =+  Raise a <$>+  validateWhitespace a ws <*>+  traverse+    (\(b, c) ->+       (,) <$>+       validateExprSyntax b <*>+       traverse+         (\(d, e) ->+            (,) <$>+            validateWhitespace a d <*>+            validateExprSyntax e)+         c)+    f+validateSimpleStatementSyntax (Return a ws expr) =+  ask `bindVM` \sctxt ->+    case _inFunction sctxt of+      Just{} ->+        Return a <$>+        validateWhitespace a ws <*>+        traverse validateExprSyntax expr+      _ -> errorVM1 (_ReturnOutsideFunction # a)+validateSimpleStatementSyntax (Expr a expr) =+  Expr a <$>+  validateExprSyntax expr+validateSimpleStatementSyntax (Assign a lvalue rs) =+  ask `bindVM` \sctxt ->+    let+      assigns =+        if isJust (_inFunction sctxt)+        then+          (lvalue : (snd <$> NonEmpty.init rs)) ^..+          folded.unvalidated.assignTargets.identValue+        else []+    in+      Assign a <$>+      validateAssignmentSyntax a lvalue <*>+      ((\a b -> case a of; [] -> pure b; a : as -> a :| (snoc as b)) <$>+       traverse+         (\(ws, b) ->+            (,) <$>+            validateEquals a ws <*>+            validateAssignmentSyntax a b)+         (NonEmpty.init rs) <*>+       (\(ws, b) -> (,) <$> validateEquals a ws <*> validateExprSyntax b)+         (NonEmpty.last rs)) <*+      liftVM0 (modify (assigns ++))+validateSimpleStatementSyntax (AugAssign a lvalue aa rvalue) =+  AugAssign a <$>+  (if canAssignTo lvalue+    then case lvalue of+      Ident{} -> validateExprSyntax lvalue+      Deref{} -> validateExprSyntax lvalue+      Subscript{} -> validateExprSyntax lvalue+      _ -> errorVM1 (_CannotAugAssignTo # (a, lvalue ^. unvalidated))+    else errorVM1 (_CannotAssignTo # (a, lvalue ^. unvalidated))) <*>+  pure aa <*>+  validateExprSyntax rvalue+validateSimpleStatementSyntax (Pass a ws) =+  Pass a <$> validateWhitespace a ws+validateSimpleStatementSyntax (Break a ws) =+  Break a <$+  (ask `bindVM` \sctxt ->+     if _inLoop sctxt+     then pure ()+     else errorVM1 (_BreakOutsideLoop # a)) <*>+  validateWhitespace a ws+validateSimpleStatementSyntax (Continue a ws) =+  Continue a <$+  (ask `bindVM` \sctxt ->+     (if _inLoop sctxt+      then pure ()+      else errorVM1 (_ContinueOutsideLoop # a)) *>+     (if _inFinally sctxt+      then errorVM1 (_ContinueInsideFinally # a)+      else pure ())) <*>+  validateWhitespace a ws+validateSimpleStatementSyntax (Global a ws ids) =+  ask `bindVM` \ctx ->+  let+    params = ctx ^.. inFunction.folded.functionParams.folded+  in+    Global a ws <$>+    traverse+      (\i ->+         let+           ival = i ^. getting identValue+         in+         (if ival `elem` params+          then errorVM1 $ _ParameterMarkedGlobal # (a, ival)+          else pure ()) *>+         validateIdentSyntax i)+      ids+validateSimpleStatementSyntax (Nonlocal a ws ids) =+  ask `bindVM` \sctxt ->+  get `bindVM` \nls ->+  (case deleteFirstsBy' (\a -> (==) (a ^. unvalidated.identValue)) (ids ^.. folded.unvalidated) nls of+     [] -> pure ()+     ids -> traverse_ (\e -> errorVM1 (_NoBindingNonlocal # e)) ids) *>+  case sctxt ^? inFunction._Just.functionParams of+    Nothing -> errorVM1 (_NonlocalOutsideFunction # a)+    Just params ->+      case intersect params (ids ^.. folded.unvalidated.identValue) of+        [] -> Nonlocal a ws <$> traverse validateIdentSyntax ids+        bad -> errorVM1 (_ParametersNonlocal # (a, bad))+validateSimpleStatementSyntax (Del a ws ids) =+  Del a ws <$>+  traverse+    (\x ->+       validateExprSyntax x <*+       if canDelete x+       then pure ()+       else errorVM1 $ _CannotDelete # (a, x ^. unvalidated))+    ids+validateSimpleStatementSyntax (Import a ws mns) =+  Import a ws <$> traverse (pure . coerce) mns+validateSimpleStatementSyntax (From a ws1 mn ws2 ts) =+  From a ws1 (coerce mn) <$>+  validateWhitespace a ws2 <*>+  validateImportTargetsSyntax ts++canDelete :: Expr v a -> Bool+canDelete None{} = False+canDelete Ellipsis{} = False+canDelete UnOp{} = False+canDelete Int{} = False+canDelete Call{} = False+canDelete BinOp{} = False+canDelete Bool{} = False+canDelete Unit{} = False+canDelete Yield{} = False+canDelete YieldFrom{} = False+canDelete Ternary{} = False+canDelete ListComp{} = False+canDelete DictComp{} = False+canDelete Dict{} = False+canDelete SetComp{} = False+canDelete Set{} = False+canDelete Lambda{} = False+canDelete Float{} = False+canDelete Imag{} = False+canDelete Not{} = False+canDelete Generator{} = False+canDelete Await{} = False+canDelete String{} = False+canDelete (Parens _ _ a _) = canDelete a+canDelete (List _ _ a _) =+  all (allOf (folded.getting _Exprs) canDelete) a &&+  not (any (\case; ListUnpack{} -> True; _ -> False) $ a ^.. folded.folded)+canDelete (Tuple _ a _ b) =+  all+    canDelete+    ((a ^?! getting _Exprs) : toListOf (folded.folded.getting _Exprs) b) &&+  not (any (\case; TupleUnpack{} -> True; _ -> False) $ a : toListOf (folded.folded) b)+canDelete Deref{} = True+canDelete Subscript{} = True+canDelete Ident{} = True++validateSmallStatementSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => SmallStatement v a+  -> ValidateSyntax e (SmallStatement (Nub (Syntax ': v)) a)+validateSmallStatementSyntax (MkSmallStatement s ss sc cmt nl) =+  (\s' ss' sc' -> MkSmallStatement s' ss' sc' cmt nl) <$>+  validateSimpleStatementSyntax s <*>+  traverse (bitraverse validateSemicolon validateSimpleStatementSyntax) ss <*>+  traverse validateSemicolon sc++validateStatementSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Statement v a+  -> ValidateSyntax e (Statement (Nub (Syntax ': v)) a)+validateStatementSyntax (CompoundStatement c) =+  CompoundStatement <$> validateCompoundStatementSyntax c+validateStatementSyntax (SmallStatement idnts a) =+  SmallStatement idnts <$> validateSmallStatementSyntax a++canAssignTo :: Expr v a -> Bool+canAssignTo None{} = False+canAssignTo Ellipsis{} = False+canAssignTo UnOp{} = False+canAssignTo Int{} = False+canAssignTo Call{} = False+canAssignTo BinOp{} = False+canAssignTo Bool{} = False+canAssignTo Unit{} = False+canAssignTo Yield{} = False+canAssignTo YieldFrom{} = False+canAssignTo Ternary{} = False+canAssignTo ListComp{} = False+canAssignTo DictComp{} = False+canAssignTo Dict{} = False+canAssignTo SetComp{} = False+canAssignTo Set{} = False+canAssignTo Lambda{} = False+canAssignTo Float{} = False+canAssignTo Imag{} = False+canAssignTo Not{} = False+canAssignTo Generator{} = False+canAssignTo Await{} = False+canAssignTo String{} = False+canAssignTo (Parens _ _ a _) = canAssignTo a+canAssignTo (List _ _ a _) =+  all (allOf (folded.getting _Exprs) canAssignTo) a+canAssignTo (Tuple _ a _ b) =+  all canAssignTo ((a ^?! getting _Exprs) : toListOf (folded.folded.getting _Exprs) b)+canAssignTo Deref{} = True+canAssignTo Subscript{} = True+canAssignTo Ident{} = True++validateArgsSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => CommaSep1' (Arg v a)+  -> ValidateSyntax e (CommaSep1' (Arg (Nub (Syntax ': v)) a))+validateArgsSyntax e = unsafeCoerce e <$ go [] False False (toList e)+  where+    go+      :: (AsSyntaxError e a, Member Indentation v)+      => [String]+      -- ^ Have we seen a keyword argument?+      -> Bool+      -- ^ Have we seen a **argument?+      -> Bool+      -> [Arg v a]+      -> ValidateSyntax e [Arg (Nub (Syntax ': v)) a]+    go _ _ _ [] = pure []+    go names False False (PositionalArg a expr : args) =+      liftA2 (:)+        (PositionalArg a <$> validateExprSyntax expr)+        (go names False False args)+    go names seenKeyword seenUnpack (PositionalArg a expr : args) =+      when seenKeyword (errorVM1 (_PositionalAfterKeywordArg # (a, expr ^. unvalidated))) *>+      when seenUnpack (errorVM1 (_PositionalAfterKeywordUnpacking # (a, expr ^. unvalidated))) *>+      go names seenKeyword seenUnpack args+    go names seenKeyword False (StarArg a ws expr : args) =+      liftA2 (:)+        (StarArg a <$> validateWhitespace a ws <*> validateExprSyntax expr)+        (go names seenKeyword False args)+    go names seenKeyword seenUnpack (StarArg a _ expr : args) =+      when seenKeyword (errorVM1 (_PositionalAfterKeywordArg # (a, expr ^. unvalidated))) *>+      when seenUnpack (errorVM1 (_PositionalAfterKeywordUnpacking # (a, expr ^. unvalidated))) *>+      go names seenKeyword seenUnpack args+    go names _ seenUnpack (KeywordArg a name ws2 expr : args)+      | _identValue name `elem` names =+          errorVM1 (_DuplicateArgument # (a, _identValue name)) <*>+          validateIdentSyntax name <*>+          go names True seenUnpack args+      | otherwise =+          liftA2 (:)+            (KeywordArg a <$>+             validateIdentSyntax name <*>+             pure ws2 <*>+             validateExprSyntax expr)+            (go (_identValue name:names) True seenUnpack args)+    go names seenKeyword _ (DoubleStarArg a ws expr : args) =+      liftA2 (:)+        (DoubleStarArg a <$>+         validateWhitespace a ws <*>+         validateExprSyntax expr)+        (go names seenKeyword True args)++newtype HaveSeenStarArg = HaveSeenStarArg Bool+newtype HaveSeenKeywordArg = HaveSeenKeywordArg Bool+newtype HaveSeenEmptyStarArg a = HaveSeenEmptyStarArg (Maybe a)++validateParamsSyntax+  :: forall e v a+   . ( AsSyntaxError e a+     , Member Indentation v+     )+  => Bool -- ^ These are the parameters to a lambda+  -> CommaSep (Param v a)+  -> ValidateSyntax e (CommaSep (Param (Nub (Syntax ': v)) a))+validateParamsSyntax isLambda e =+  unsafeCoerce e <$+  go+    []+    (HaveSeenStarArg False)+    (HaveSeenEmptyStarArg Nothing)+    (HaveSeenKeywordArg False)+    (toList e)+  where+    checkTy+      :: a+      -> Maybe (Colon, Expr v a)+      -> ValidateSyntax e (Maybe (Colon, Expr (Nub (Syntax ': v)) a))+    checkTy a mty =+      if isLambda+      then traverse (\_ -> errorVM1 (_TypedParamInLambda # a)) mty+      else traverseOf (traverse._2) validateExprSyntax mty++    go+      :: [String] -- identifiers that we've seen+      -> HaveSeenStarArg -- have we seen a star argument?+      -> HaveSeenEmptyStarArg a -- have we seen an empty star argument?+      -> HaveSeenKeywordArg -- have we seen a keyword parameter?+      -> [Param v a]+      -> ValidateSyntax e [Param (Nub (Syntax ': v)) a]+    go _ _ (HaveSeenEmptyStarArg b) _ [] =+      case b of+        Nothing -> pure []+        Just b' -> errorVM1 $ _NoKeywordsAfterEmptyStarArg # b'+    go names bsa besa bkw@(HaveSeenKeywordArg False) (PositionalParam a name mty : params)+      | _identValue name `elem` names =+          errorVM1 (_DuplicateArgument # (a, _identValue name)) <*>+          validateIdentSyntax name <*>+          checkTy a mty <*>+          go (_identValue name:names) bsa besa bkw params+      | otherwise =+          liftA2+            (:)+            (PositionalParam a <$>+             validateIdentSyntax name <*>+             checkTy a mty)+            (go (_identValue name:names) bsa besa bkw params)+    go names (HaveSeenStarArg b) besa bkw (StarParam a _ name mty : params)+      | _identValue name `elem` names =+          if b+          then+            errorVM1 (_ManyStarredParams # a) <*>+            errorVM1 (_DuplicateArgument # (a, _identValue name)) <*>+            validateIdentSyntax name <*>+            checkTy a mty <*>+            go+              (_identValue name:names)+              (HaveSeenStarArg True)+              besa+              bkw+              params+          else+            errorVM1 (_DuplicateArgument # (a, _identValue name)) <*>+            validateIdentSyntax name <*>+            checkTy a mty <*>+            go+              (_identValue name:names)+              (HaveSeenStarArg True)+              besa+              bkw+              params+      | otherwise =+          if b+          then+            errorVM1 (_ManyStarredParams # a) <*>+            validateIdentSyntax name *>+            checkTy a mty *>+            go+              (_identValue name:names)+              (HaveSeenStarArg True)+              besa+              bkw+              params+          else+            validateIdentSyntax name *>+            checkTy a mty *>+            go+              (_identValue name:names)+              (HaveSeenStarArg True)+              besa+              bkw+              params+    go names (HaveSeenStarArg b) _ bkw (UnnamedStarParam a _ : params) =+      if b+      then+        errorVM1 (_ManyStarredParams # a) <*>+        go+          names+          (HaveSeenStarArg True)+          (HaveSeenEmptyStarArg $ Just a)+          bkw+          params+      else+        go+          names+          (HaveSeenStarArg True)+          (HaveSeenEmptyStarArg $ Just a)+          bkw+          params+    go names bsa besa bkw@(HaveSeenKeywordArg True) (PositionalParam a name mty : params) =+      let+        name' = _identValue name+        errs =+          foldr (<|)+            (_PositionalAfterKeywordParam # (a, name') :| [])+            [_DuplicateArgument # (a, name') | name' `elem` names]+      in+        errorVM errs <*>+        checkTy a mty <*>+        go (name':names) bsa besa bkw params+    go names bsa _ _ (KeywordParam a name mty ws2 expr : params)+      | _identValue name `elem` names =+          errorVM1 (_DuplicateArgument # (a, _identValue name)) <*>+          checkTy a mty <*>+          go names bsa (HaveSeenEmptyStarArg Nothing) (HaveSeenKeywordArg True) params+      | otherwise =+          liftA2 (:)+            (KeywordParam a <$>+             validateIdentSyntax name <*>+             checkTy a mty <*>+             pure ws2 <*>+             validateExprSyntax expr)+            (go+               (_identValue name:names)+               bsa+               (HaveSeenEmptyStarArg Nothing)+               (HaveSeenKeywordArg True)+               params)+    go names bsa besa bkw [DoubleStarParam a ws name mty]+      | _identValue name `elem` names =+          errorVM1 (_DuplicateArgument # (a, _identValue name)) <*>+          checkTy a mty <*+          go names bsa besa bkw []+      | otherwise =+          fmap pure $+          DoubleStarParam a ws <$>+          validateIdentSyntax name <*>+          checkTy a mty <*+          go names bsa besa bkw []+    go names bsa besa bkw (DoubleStarParam a _ name mty : _) =+      (if _identValue name `elem` names+       then errorVM1 (_DuplicateArgument # (a, _identValue name))+       else pure ()) *>+      errorVM1 (_UnexpectedDoubleStarParam # (a, _identValue name)) <*>+      checkTy a mty <*+      go names bsa besa bkw []++validateModuleSyntax+  :: ( AsSyntaxError e a+     , Member Indentation v+     )+  => Module v a+  -> ValidateSyntax e (Module (Nub (Syntax ': v)) a)+validateModuleSyntax m =+  case m of+    ModuleEmpty -> pure ModuleEmpty+    ModuleBlankFinal a -> ModuleBlankFinal <$> validateBlankSyntax a+    ModuleBlank a b c ->+      (\a' -> ModuleBlank a' b) <$>+      validateBlankSyntax a <*>+      validateModuleSyntax c+    ModuleStatement a b ->+     ModuleStatement <$>+     validateStatementSyntax a <*>+     validateModuleSyntax b
+ src/Language/Python/Validate/Syntax/Error.hs view
@@ -0,0 +1,63 @@+{-# language DataKinds, KindSignatures #-}+{-# language MultiParamTypeClasses, TemplateHaskell, FunctionalDependencies,+    FlexibleInstances #-}++{-|+Module      : Language.Python.Validate.Syntax.Error+Copyright   : (C) CSIRO 2017-2018+License     : BSD3+Maintainer  : Isaac Elliott <isaace71295@gmail.com>+Stability   : experimental+Portability : non-portable+-}++module Language.Python.Validate.Syntax.Error where++import Control.Lens.TH+import Language.Python.Syntax.Expr (Expr)+import Language.Python.Syntax.Ident (Ident)++data SyntaxError a+  = PositionalAfterKeywordArg a (Expr '[] a)+  | PositionalAfterKeywordUnpacking a (Expr '[] a)+  | CannotAssignTo a (Expr '[] a)+  | CannotDelete a (Expr '[] a)+  | CannotAugAssignTo a (Expr '[] a)+  | NoBindingNonlocal (Ident '[] a)+  | PositionalAfterKeywordParam a String+  | UnexpectedDoubleStarParam a String+  | DuplicateArgument a String+  | UnexpectedNewline a+  | UnexpectedComment a+  | IdentifierReservedWord a String+  | EmptyIdentifier a+  | BadCharacter a String+  | BreakOutsideLoop a+  | ContinueOutsideLoop a+  | ReturnOutsideFunction a+  | NonlocalOutsideFunction a+  | ParametersNonlocal a [String]+  | Can'tJoinStringAndBytes a+  | YieldOutsideGenerator a+  | MalformedDecorator a+  | InvalidDictUnpacking a+  | InvalidSetUnpacking a+  | TypedParamInLambda a+  | AsyncWithOutsideCoroutine a+  | AsyncForOutsideCoroutine a+  | YieldFromInsideCoroutine a+  | YieldInsideCoroutine a+  | AwaitOutsideCoroutine a+  | AwaitInsideComprehension a+  | NullByte a+  | NonAsciiInBytes a Char+  | DefaultExceptMustBeLast a+  | WildcardImportInDefinition a+  | NoKeywordsAfterEmptyStarArg a+  | ManyStarredTargets a+  | ManyStarredParams a+  | ContinueInsideFinally a+  | ParameterMarkedGlobal a String+  deriving (Eq, Show)++makeClassyPrisms ''SyntaxError
+ test/DSL.hs view
@@ -0,0 +1,312 @@+{-# language OverloadedStrings, TemplateHaskell #-}+module DSL (dslTests) where++import Hedgehog++import Control.Lens.Fold ((^?))+import Control.Lens.Setter ((.~), over)+import Data.Function ((&))++import Language.Python.DSL+import Language.Python.Optics+import Language.Python.Render (showExpr)+import Language.Python.Syntax.CommaSep (CommaSep(..))+import Language.Python.Syntax.Punctuation (Comma(..))+import Language.Python.Syntax.Whitespace (Whitespace(..), Indents(..))++dslTests :: Group+dslTests = $$discover++prop_subscript_1 :: Property+prop_subscript_1 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (int_ 1)+    showExpr expr  === "a[1]"++prop_subscript_2 :: Property+prop_subscript_2 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, ti_ $ int_ 2])+    showExpr expr  === "a[1, 2]"++prop_subscript_3 :: Property+prop_subscript_3 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (tuple_ [ti_ $ int_ 1, s_ $ var_ "b"])+    showExpr expr  === "a[(1, *b)]"++prop_subscript_4 :: Property+prop_subscript_4 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (tuple_ [ti_ $ int_ 1])+    showExpr expr  === "a[1,]"++prop_subscript_5 :: Property+prop_subscript_5 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (tuple_ [s_ $ var_ "b"])+    showExpr expr  === "a[((*b),)]"++prop_subscript_6 :: Property+prop_subscript_6 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (slice_ Nothing Nothing Nothing)+    showExpr expr  === "a[:]"++prop_subscript_7 :: Property+prop_subscript_7 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (tuple_ [ti_ $ slice_ Nothing Nothing Nothing])+    showExpr expr  === "a[:,]"++prop_subscript_8 :: Property+prop_subscript_8 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (tuple_ [ti_ $ tuple_ [ti_ $ slice_ Nothing Nothing Nothing]])+    showExpr expr  === "a[(slice(None, None, None),),]"++prop_subscript_9 :: Property+prop_subscript_9 =+  withTests 1 . property $ do+    let+      expr =+        subs_ (var_ "a") $+        tuple_ [ti_ $ slice_ Nothing Nothing Nothing, ti_ $ slice_ Nothing Nothing Nothing]+    showExpr expr  === "a[:, :]"++prop_subscript_10 :: Property+prop_subscript_10 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (slice_ (Just $ int_ 1) Nothing Nothing)+    showExpr expr  === "a[1:]"++prop_subscript_11 :: Property+prop_subscript_11 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (slice_ (Just $ int_ 1) (Just $ int_ 1) Nothing)+    showExpr expr  === "a[1:1]"++prop_subscript_12 :: Property+prop_subscript_12 =+  withTests 1 . property $ do+    let+      expr = subs_ (var_ "a") (slice_ (Just $ int_ 1) (Just $ int_ 1) (Just $ int_ 1))+    showExpr expr  === "a[1:1:1]"++prop_subscript_13 :: Property+prop_subscript_13 =+  withTests 1 . property $ do+    let+      expr =+        subs_ (var_ "a") $+        tuple_+          [ ti_ $ slice_ (Just $ int_ 1) Nothing Nothing+          , ti_ $ slice_ (Just $ int_ 2) Nothing Nothing+          ]+    showExpr expr  === "a[1:, 2:]"++prop_subscript_14 :: Property+prop_subscript_14 =+  withTests 1 . property $ do+    let+      expr =+        subs_ (var_ "a") $+        tuple_+          [ ti_ $ slice_ (Just $ int_ 1) (Just $ int_ 1) Nothing+          , ti_ $ slice_ (Just $ int_ 2) Nothing Nothing+          ]+    showExpr expr  === "a[1:1, 2:]"++prop_subscript_15 :: Property+prop_subscript_15 =+  withTests 1 . property $ do+    let+      expr =+        subs_ (var_ "a") $+        tuple_+          [ ti_ $ slice_ (Just $ int_ 1) (Just $ int_ 1) (Just $ int_ 1)+          , ti_ $ slice_ (Just $ int_ 2) Nothing Nothing+          ]+    showExpr expr  === "a[1:1:1, 2:]"++prop_subscript_16 :: Property+prop_subscript_16 =+  withTests 1 . property $ do+    let+      expr =+        subs_ (var_ "a") $+        tuple_+        [ ti_ $+          tuple_+            [ ti_ $ slice_ (Just $ int_ 1) Nothing Nothing+            , ti_ $ int_ 2+            ]+        ]+    showExpr expr  === "a[(slice(1, None, None), 2),]"++prop_parameters_1 :: Property+prop_parameters_1 =+  withTests 1 . property $ do+    let+      st = def_ "a" [] [line_ pass_]++      params1 =+        CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $+        CommaSepMany (p_ "test2") (MkComma $ replicate 3 Space) $+        CommaSepNone+      st1 = st & _Fundef.fdParameters .~ params1++      params2 =+        CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $+        CommaSepMany (p_ "test4") (MkComma $ replicate 3 Space) $+        CommaSepNone+      st2 = st & _Fundef.fdParameters .~ params2++    (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4"]) === st2++prop_parameters_2 :: Property+prop_parameters_2 =+  withTests 1 . property $ do+    let+      st = def_ "a" [] [line_ pass_]++      params1 =+        CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $+        CommaSepMany (p_ "test2") (MkComma $ replicate 3 Space) $+        CommaSepNone+      st1 = st & _Fundef.fdParameters .~ params1++      params2 =+        CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $+        CommaSepMany (p_ "test4") (MkComma $ replicate 3 Space) $+        CommaSepOne (p_ "test5")+      st2 = st & _Fundef.fdParameters .~ params2++    (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4", p_ "test5"]) === st2++prop_parameters_3 :: Property+prop_parameters_3 =+  withTests 1 . property $ do+    let+      st = def_ "a" [] [line_ pass_]++      params1 =+        CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $+        CommaSepMany (p_ "test2") (MkComma $ replicate 3 Space) $+        CommaSepNone+      st1 = st & _Fundef.fdParameters .~ params1++      params2 = CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) CommaSepNone+      st2 = st & _Fundef.fdParameters .~ params2++    (st1 & _Fundef.parameters_ .~ [p_ "test3"]) === st2++prop_parameters_4 :: Property+prop_parameters_4 =+  withTests 1 . property $ do+    let+      st = def_ "a" [] [line_ pass_]++      params1 =+        CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $+        CommaSepOne (p_ "test2")+      st1 = st & _Fundef.fdParameters .~ params1++      params2 =+        CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $+        CommaSepOne (p_ "test4")+      st2 = st & _Fundef.fdParameters .~ params2++    (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4"]) === st2++prop_parameters_5 :: Property+prop_parameters_5 =+  withTests 1 . property $ do+    let+      st = def_ "a" [] [line_ pass_]++      params1 =+        CommaSepMany (p_ "test1") (MkComma $ replicate 5 Space) $+        CommaSepOne (p_ "test2")+      st1 = st & _Fundef.fdParameters .~ params1++      params2 =+        CommaSepMany (p_ "test3") (MkComma $ replicate 5 Space) $+        CommaSepMany (p_ "test4") (MkComma [Space]) $+        CommaSepOne (p_ "test5")+      st2 = st & _Fundef.fdParameters .~ params2++    (st1 & _Fundef.parameters_ .~ [p_ "test3", p_ "test4", p_ "test5"]) === st2++prop_body_1 :: Property+prop_body_1 =+  withTests 1 . property $ do+    let+      st = def_ "a" [] [line_ pass_]++    st ^? _Fundef.fdIndents === Just (Indents [] ())+    over (_Fundef.body_) id st === st++prop_body_2 :: Property+prop_body_2 =+  withTests 1 . property $ do+    let+      st = def_ "a" [] [line_ pass_]++    st ^? _Fundef.body_ === Just [line_ pass_]++prop_body_3 :: Property+prop_body_3 =+  withTests 1 . property $ do+    let+      stInner = def_ "b" [] [line_ pass_]+      stOuter = def_ "a" [] [line_ pass_, line_ stInner]++      newIndent = replicate 10 Space++    (stOuter & _Indent .~ newIndent) ^? _Fundef.body_ ===+      Just+      [ line_ $ pass_ & _Indent .~ newIndent+      , line_ $ stInner & _Indent .~ newIndent+      ]++prop_body_4 :: Property+prop_body_4 =+  withTests 1 . property $ do+    let+      newIndent = replicate 10 Space++      stInner = def_ "b" [] [line_ pass_]++      outerBody =+        [ line_ pass_+        , line_ stInner+        ]++      outerBody' =+        [ line_ $ pass_ & _Indent .~ newIndent+        , line_ $ stInner & _Indent .~ newIndent+        ]++      stOuter = def_ "a" [] outerBody & _Indent .~ newIndent++      finalBody =+        [ line_ pass_+        , line_ $ stInner & _Indent .~ newIndent+        , line_ pass_+        ]++      stFinal' = def_ "a" [] finalBody & _Indent .~ newIndent++    stOuter ^? _Fundef.body_ === Just outerBody'+    (stOuter & _Fundef.body_ .~ finalBody) === stFinal'
+ test/Helpers.hs view
@@ -0,0 +1,150 @@+{-# language DataKinds #-}+{-# language RankNTypes #-}+{-# language ScopedTypeVariables #-}+module Helpers where++import Hedgehog++import Control.Lens.Fold ((^?), folded)+import Control.Monad (void)+import Data.List.NonEmpty (NonEmpty(..))+import Data.Semigroup (Semigroup)+import Data.Text (Text)+import Data.Validation (Validation(..), _Failure)+import Text.Megaparsec.Pos (SourcePos(..), mkPos)++import Language.Python.Internal.Lexer+  (SrcInfo, insertTabs, tokenize+  )+import Language.Python.Internal.Token (PyToken)+import Language.Python.Parse (Parser)+import Language.Python.Parse.Error (ParseError, ErrorItem(..), _ParseError)+import Language.Python.Internal.Parse (runParser)+import Language.Python.Syntax.Expr (Expr)+import Language.Python.Syntax.Module (Module)+import Language.Python.Syntax.Statement (Statement)+import Language.Python.Validate++doTokenize :: Monad m => Text -> PropertyT m [PyToken SrcInfo]+doTokenize input =+  case tokenize "test" input of+    Left err -> annotateShow (err :: ParseError SrcInfo) *> failure+    Right a -> pure a++doTabs+  :: forall ann m+   . (Semigroup ann, Show ann, Monad m)+  => ann+  -> [PyToken ann]+  -> PropertyT m [PyToken ann]+doTabs ann input =+  case insertTabs ann input of+    Left err -> annotateShow (err :: ParseError ann) *> failure+    Right a -> pure a++doParse :: Monad m => Parser a -> [PyToken SrcInfo] -> PropertyT m a+doParse pa input = do+  let res = runParser "test" pa input+  case res of+    Left err -> do+      annotateShow (err :: ParseError SrcInfo)+      failure+    Right a -> pure a++syntaxValidateModule+  :: Module '[] ()+  -> PropertyT IO+       (Validation+          (NonEmpty (SyntaxError ()))+          (Module '[Syntax, Indentation] ()))+syntaxValidateModule x =+  case runValidateIndentation $ validateModuleIndentation x of+    Failure errs -> do+      annotateShow (errs :: NonEmpty (IndentationError ()))+      failure+    Success a ->+      pure $ runValidateSyntax (validateModuleSyntax a)++syntaxValidateStatement+  :: Statement '[] ()+  -> PropertyT IO+       (Validation+          (NonEmpty (SyntaxError ()))+          (Statement '[Syntax, Indentation] ()))+syntaxValidateStatement x =+  case runValidateIndentation $ validateStatementIndentation x of+    Failure errs -> do+      annotateShow (errs :: NonEmpty (IndentationError ()))+      failure+    Success a ->+      pure $ runValidateSyntax (validateStatementSyntax a)++syntaxValidateExpr+  :: Expr '[] ()+  -> PropertyT IO+       (Validation+          (NonEmpty (SyntaxError ()))+          (Expr '[Syntax, Indentation] ()))+syntaxValidateExpr x =+  case runValidateIndentation $ validateExprIndentation x of+    Failure errs -> do+      annotateShow (errs :: NonEmpty (IndentationError ()))+      failure+    Success a ->+      pure $ runValidateSyntax (validateExprSyntax a)++shouldBeFailure :: MonadTest m => Validation e a -> m ()+shouldBeFailure res =+  case res of+    Success{} -> failure+    Failure{} -> success++shouldBeSuccess :: (MonadTest m, Show e) => Validation e a -> m a+shouldBeSuccess res =+  case res of+    Success a -> pure a+    Failure err -> do+      annotateShow err+      failure++shouldBeParseSuccess+  :: MonadTest m+  => (FilePath -> Text -> Validation (NonEmpty (ParseError SrcInfo)) a)+  -> Text -> m a+shouldBeParseSuccess p = shouldBeSuccess . p "test"++shouldBeParseFailure+  :: MonadTest m+  => (FilePath -> Text -> Validation (NonEmpty (ParseError SrcInfo)) a)+  -> Text -> m ()+shouldBeParseFailure p = shouldBeFailure . p "test"++shouldBeParseError+  :: (MonadTest m, Show e, Show a)+  => Int+  -> Int+  -> PyToken ()+  -> Validation (NonEmpty (ParseError e)) a+  -> m ()+shouldBeParseError line col tk res =+  case res ^? _Failure.folded._ParseError of+    Just (srcPos :| _, Just (Tokens (errorItem :| [])), _) -> do+      sourceLine srcPos === mkPos line+      sourceColumn srcPos === mkPos col++      void errorItem === tk+    _ -> do+      annotateShow res+      failure++shouldBeSyntaxError+  :: (MonadTest m, Show a)+  => SyntaxError ()+  -> Validation (NonEmpty (SyntaxError ())) a+  -> m ()+shouldBeSyntaxError err res =+  case res ^? _Failure.folded of+    Just err' -> err === err'+    _ -> do+      annotateShow res+      failure
+ test/LexerParser.hs view
@@ -0,0 +1,543 @@+{-# language OverloadedStrings, OverloadedLists, TemplateHaskell #-}+module LexerParser (lexerParserTests) where++import Hedgehog+import Control.Monad (void)+import Data.List.NonEmpty (NonEmpty(..))+import qualified Data.Text as Text++import Language.Python.DSL+import Language.Python.Render+import Language.Python.Parse (parseModule, parseStatement, parseExpr, parseExprList)+import Language.Python.Syntax.CommaSep (CommaSep(..), Comma(..))+import Language.Python.Syntax.Expr (Expr(..))+import Language.Python.Syntax.Strings+  ( StringLiteral(..), StringType(..), QuoteType(..), PyChar(..)+  , RawBytesPrefix(..), RawStringPrefix(..)+  )+import Language.Python.Syntax.Whitespace (Whitespace(..))++import Helpers (shouldBeParseSuccess, shouldBeParseFailure)++lexerParserTests :: Group+lexerParserTests = $$discover++prop_fulltrip_1 :: Property+prop_fulltrip_1 =+  withTests 1 . property $ do+    let str = "def a(x, y=2, *z, **w):\n   return 2 + 3"++    tree <- shouldBeParseSuccess parseStatement str++    showStatement tree === str++prop_fulltrip_2 :: Property+prop_fulltrip_2 =+  withTests 1 . property $ do+    let str = "(   1\n       *\n  3\n    )"++    tree <- shouldBeParseSuccess parseExpr str++    showExpr tree === str++prop_fulltrip_3 :: Property+prop_fulltrip_3 =+  withTests 1 . property $ do+    let str = "pass;"++    tree <- shouldBeParseSuccess parseStatement str++    showStatement tree === str++prop_fulltrip_4 :: Property+prop_fulltrip_4 =+  withTests 1 . property $ do+    let str = "def a():\n pass\n #\n pass\n"++    tree <- shouldBeParseSuccess parseStatement str++    showStatement tree === str++prop_fulltrip_5 :: Property+prop_fulltrip_5 =+  withTests 1 . property $ do+    let str = "if False:\n pass\n pass\nelse:\n pass\n pass\n"++    tree <- shouldBeParseSuccess parseStatement str++    showStatement tree === str++prop_fulltrip_6 :: Property+prop_fulltrip_6 =+  withTests 1 . property $ do+    let str = "# blah\ndef boo():\n    pass\n       #bing\n    #   bop\n"++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_7 :: Property+prop_fulltrip_7 =+  withTests 1 . property $ do+    let str = "if False:\n pass\nelse \\\n      \\\r\n:\n pass\n"++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_8 :: Property+prop_fulltrip_8 =+  withTests 1 . property $ do+    let str = "def a():\n \n pass\n pass\n"++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_9 :: Property+prop_fulltrip_9 =+  withTests 1 . property $ do+    let+      str =+        "try:\n pass\nexcept False:\n pass\nelse:\n pass\nfinally:\n pass\n def a():\n  pass\n pass\n"++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_10 :: Property+prop_fulltrip_10 =+  withTests 1 . property $ do+    let+      str =+        Text.unlines+        [ "from blah import  boo"+        , "import baz   as wop"+        , ""+        , "def thing():"+        , "    pass"+        , ""+        , "def    hello():"+        , "    what; up;"+        , ""+        , "def boo(a, *b, c=1, **d):"+        , "    pass"+        ]++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_11 :: Property+prop_fulltrip_11 =+  withTests 1 . property $ do+    let+      str =+        Text.unlines+        [ "if False:"+        , " pass"+        , " pass"+        , "else:"+        , " \tpass"+        , " \tpass"+        ]++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_12 :: Property+prop_fulltrip_12 =+  withTests 1 . property $ do+    let+      str =+        Text.unlines+        [ "try:"+        , " \tpass"+        , " \tdef a():"+        , " \t pass"+        , " \tpass"+        , "finally:"+        , " pass"+        ]++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_13 :: Property+prop_fulltrip_13 =+  withTests 1 . property $ do+    let+      str =+        Text.unlines+        [ "if []:"+        , " False"+        , " def a():"+        , "  pass"+        , "  pass"+        , ""+        , "else:"+        , " pass"+        , " pass"+        ]++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_14 :: Property+prop_fulltrip_14 =+  withTests 1 . property $ do+    let+      str = "not ((False for a in False) if False else False or False)"++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_15 :: Property+prop_fulltrip_15 =+  withTests 1 . property $ do+    let+      str = "01."++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_16 :: Property+prop_fulltrip_16 =+  withTests 1 . property $ do+    let+      str = "def a():\n  return ~i"++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_17 :: Property+prop_fulltrip_17 =+  withTests 1 . property $ do+    let str = "r\"\\\"\""++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_18 :: Property+prop_fulltrip_18 =+  withTests 1 . property $ do+    let str = "\"\0\""++    tree <- shouldBeParseSuccess parseModule str++    showModule tree === str++prop_fulltrip_19 :: Property+prop_fulltrip_19 =+  withTests 1 . property $ do+    let str = " \\\n"++    shouldBeParseFailure parseModule str++prop_fulltrip_20 :: Property+prop_fulltrip_20 =+  withTests 1 . property $ do+    let str = " pass"++    shouldBeParseFailure parseModule str++prop_fulltrip_21 :: Property+prop_fulltrip_21 =+  withTests 1 . property $ do+    let str = "if a:\n  \\\n\n  pass"++    shouldBeParseFailure parseModule str++prop_fulltrip_22 :: Property+prop_fulltrip_22 =+  withTests 1 . property $ do+    let str = "for a in (b, *c): pass"++    void $ shouldBeParseSuccess parseModule str++prop_fulltrip_23 :: Property+prop_fulltrip_23 =+  withTests 1 . property $ do+    let str = "None,*None"++    void $ shouldBeParseSuccess parseModule str++prop_fulltrip_24 :: Property+prop_fulltrip_24 =+  withTests 1 . property $ do+    let str = "'\1'"++    void $ shouldBeParseSuccess parseModule str++prop_fulltrip_25 :: Property+prop_fulltrip_25 =+  withTests 1 . property $ do+    let str = "'\11'"++    void $ shouldBeParseSuccess parseModule str++prop_fulltrip_26 :: Property+prop_fulltrip_26 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawBytesLiteral ()+               Prefix_br+               LongString+               SingleQuote+               [ Char_esc_bslash ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_27 :: Property+prop_fulltrip_27 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               LongString+               SingleQuote+               [ Char_lit '\\', Char_lit '\\', Char_lit '\\', Char_lit '\'' ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_28 :: Property+prop_fulltrip_28 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               ShortString+               DoubleQuote+               [ Char_lit '\\' ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_29 :: Property+prop_fulltrip_29 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               ShortString+               DoubleQuote+               [ Char_lit '\\', Char_lit '\\' ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_30 :: Property+prop_fulltrip_30 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               ShortString+               DoubleQuote+               [ Char_lit '\\', Char_lit '\\', Char_lit '\\' ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_31 :: Property+prop_fulltrip_31 =+  withTests 1 . property $ do+    let str = "del(a)"++    void $ shouldBeParseSuccess parseModule str++prop_fulltrip_32 :: Property+prop_fulltrip_32 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               LongString+               DoubleQuote+               [ Char_lit ' ', Char_lit '"' ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_33 :: Property+prop_fulltrip_33 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               LongString+               DoubleQuote+               [ Char_lit '"', Char_lit ' ' ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_34 :: Property+prop_fulltrip_34 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               LongString+               DoubleQuote+               [ Char_lit '"' ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_35 :: Property+prop_fulltrip_35 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (pure $+             RawStringLiteral ()+               Prefix_r+               LongString+               DoubleQuote+               [ Char_lit '\\'+               , Char_esc_bslash+               , Char_esc_doublequote+               ]+               [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_36 :: Property+prop_fulltrip_36 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (RawStringLiteral ()+               Prefix_r+               LongString+               SingleQuote+               [Char_lit '\\', Char_esc_bslash] [] :|+            [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_37 :: Property+prop_fulltrip_37 =+  withTests 1 . property $ do+    let str = "None,*None"++    void $ shouldBeParseSuccess parseExprList str++prop_fulltrip_38 :: Property+prop_fulltrip_38 =+  withTests 1 . property $ do+    let str =+          showExpr $+          String ()+            (RawStringLiteral ()+               Prefix_r+               LongString+               SingleQuote+               [Char_esc_bslash, Char_lit '\\'] [] :|+            [])+    annotateShow str++    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)++prop_fulltrip_39 :: Property+prop_fulltrip_39 =+  withTests 1 . property $ do+    let str = "def a(*b, *): pass"++    shouldBeParseFailure parseStatement str++prop_fulltrip_40 :: Property+prop_fulltrip_40 =+  withTests 1 . property $ do+    let str = "def a():\n    yield op, oparg"+    res <- shouldBeParseSuccess parseStatement str+    str === showStatement (() <$ res)++prop_fulltrip_41 :: Property+prop_fulltrip_41 =+  withTests 1 . property $ do+    let+      s = "def a(*a, *b): pass"+    shouldBeParseFailure parseModule s++prop_fulltrip_42 :: Property+prop_fulltrip_42 =+  withTests 1 . property $ do+    let+      s = "lambda *a, *b: pass"+    shouldBeParseFailure parseModule s++prop_fulltrip_43 :: Property+prop_fulltrip_43 =+  withTests 1 . property $ do+    let+      e =+        Yield+        { _unsafeExprAnn = ()+        , _unsafeYieldWhitespace = [Space]+        , _unsafeYieldValue =+            CommaSepMany (Ident (MkIdent () "a" [])) (MkComma [Space]) $+            CommaSepMany (tuple_ [ti_ $ var_ "b"]) (MkComma []) $+            CommaSepNone+        }+      -- yield a, (b,),+      str = showExpr e+    res <- shouldBeParseSuccess parseExpr str+    str === showExpr (() <$ res)
+ test/Main.hs view
@@ -0,0 +1,25 @@+{-# options_ghc -fno-warn-unused-do-bind #-}+{-# language DataKinds, TypeOperators, FlexibleContexts #-}+{-# language OverloadedStrings #-}+module Main where++import DSL+import LexerParser+import Optics+import Parser+import Roundtrip+import Scope+import Syntax++import Control.Monad (when)+import System.Exit++import Hedgehog++main :: IO ()+main = do+  results <- traverse checkParallel groups+  when (not (and results))+    exitFailure+  where+    groups = [lexerParserTests, dslTests, parserTests, opticsTests, scopeTests, syntaxTests, roundtripTests]
+ test/Optics.hs view
@@ -0,0 +1,43 @@+{-# language OverloadedStrings, TemplateHaskell #-}+module Optics (opticsTests) where++import Hedgehog++import Control.Lens.Plated (transformOn)+import Control.Lens.Setter ((.~))+import Control.Monad.IO.Class (liftIO)+import qualified Data.Text.IO as Text++import Language.Python.Parse (parseModule)+import Language.Python.Render (showModule)+import Language.Python.Syntax.Statement (_Statements)+import Language.Python.Syntax.Whitespace (Whitespace (..))+import Language.Python.Optics (_Indent)++import Helpers (shouldBeParseSuccess)++opticsTests :: Group+opticsTests = $$discover++prop_optics_1 :: Property+prop_optics_1 =+  withTests 1 . property $ do+    str <- liftIO $ Text.readFile "test/files/indent_optics_in.py"++    tree <- shouldBeParseSuccess parseModule str++    str' <- liftIO $ Text.readFile "test/files/indent_optics_out.py"+    showModule+      (transformOn _Statements (_Indent .~ [Space, Space, Space, Space]) tree) === str'++prop_optics_2 :: Property+prop_optics_2 =+  withTests 1 . property $ do+    str <- liftIO $ Text.readFile "test/files/indent_optics_in2.py"++    tree <- shouldBeParseSuccess parseModule str+    -- annotateShow $! tree++    str' <- liftIO $ Text.readFile "test/files/indent_optics_out2.py"+    showModule+      (transformOn _Statements (_Indent .~ [Space, Space, Space, Space]) tree) === str'
+ test/Parser.hs view
@@ -0,0 +1,21 @@+{-# language OverloadedStrings, TemplateHaskell #-}+module Parser (parserTests) where++import Hedgehog++import Language.Python.Internal.Token (PyToken(..))+import Language.Python.Parse (parseStatement)++import Helpers (shouldBeParseError)++parserTests :: Group+parserTests = $$discover++prop_parser_1 :: Property+prop_parser_1 =+  withTests 1 . property $ do+    let+      e = "for x in a, *b: pass"+      res = parseStatement "test" e++    shouldBeParseError 1 13 (TkStar ()) res
+ test/Roundtrip.hs view
@@ -0,0 +1,83 @@+{-# language OverloadedStrings #-}+{-# language DataKinds #-}+module Roundtrip (roundtripTests) where++import Control.Monad.IO.Class (liftIO)+import Data.List.NonEmpty (NonEmpty)+import Data.String (fromString)+import Data.Text (Text)+import Data.Validation (Validation(..))+import Hedgehog+  ( (===), Group(..), Property, PropertyT, annotateShow, failure, property+  , withTests, withShrinks+  )+import System.FilePath ((</>))++import qualified Data.Text.IO as StrictText++import Language.Python.Internal.Lexer (SrcInfo)+import Language.Python.Render (showModule)+import Language.Python.Parse (parseModule)+import Language.Python.Validate+  ( IndentationError, SyntaxError+  , runValidateIndentation, validateModuleIndentation, runValidateSyntax+  , validateModuleSyntax+  )++import Helpers (shouldBeParseSuccess)++roundtripTests :: Group+roundtripTests =+  Group "Roundtrip tests" $+  (\name -> (fromString name, withTests 1 . withShrinks 0 $ doRoundtripFile name)) <$>+  [ "decorators.py"+  , "string.py"+  , "set.py"+  , "regex.py"+  , "asyncstatements.py"+  , "typeann.py"+  , "dictcomp.py"+  , "imaginary.py"+  , "weird.py"+  , "weird2.py"+  , "django.py"+  , "django2.py"+  , "test.py"+  , "ansible.py"+  , "comments.py"+  , "pypy.py"+  , "pypy2.py"+  , "sqlalchemy.py"+  , "numpy.py"+  , "numpy2.py"+  , "mypy.py"+  , "mypy2.py"+  , "requests.py"+  , "requests2.py"+  , "joblib.py"+  , "joblib2.py"+  , "pandas.py"+  , "pandas2.py"+  ]++doRoundtripFile :: FilePath -> Property+doRoundtripFile name =+  property $ do+    file <- liftIO . StrictText.readFile $ "test/files" </> name+    doRoundtrip file++doRoundtrip :: Text -> PropertyT IO ()+doRoundtrip file = do+  py <- shouldBeParseSuccess parseModule file+  case runValidateIndentation $ validateModuleIndentation py of+    Failure errs -> do+      annotateShow (errs :: NonEmpty (IndentationError SrcInfo))+      failure+    Success res ->+      case runValidateSyntax (validateModuleSyntax res) of+        Failure errs' -> do+          annotateShow (errs' :: NonEmpty (SyntaxError SrcInfo))+          failure+        Success _ -> do+          annotateShow py+          showModule py === file
+ test/Scope.hs view
@@ -0,0 +1,192 @@+{-# language OverloadedStrings, DataKinds, TemplateHaskell #-}+module Scope (scopeTests) where++import Hedgehog++import Control.Lens ((#), has)+import Data.Function ((&))+import Data.Functor (($>))+import Data.List.NonEmpty (NonEmpty ((:|)))+import Data.Validation (Validation(..), _Success)++import Language.Python.Validate+import Language.Python.DSL+import Language.Python.Optics+import Language.Python.Syntax.Whitespace++scopeTests :: Group+scopeTests = $$discover++fullyValidate+  :: Statement '[] ()+  -> PropertyT IO+       (Validation+          (NonEmpty (ScopeError ()))+          (Statement '[Scope, Syntax, Indentation] ()))+fullyValidate x =+  case runValidateIndentation $ validateStatementIndentation x of+    Failure errs -> do+      annotateShow (errs :: NonEmpty (IndentationError ()))+      failure+    Success a ->+      case runValidateSyntax (validateStatementSyntax a) of+        Failure errs -> do+          annotateShow (errs :: NonEmpty (SyntaxError ()))+          failure+        Success a' -> pure $ runValidateScope (validateStatementScope a')++prop_scope_1 :: Property+prop_scope_1 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" [p_ "a", p_ "b"]+          [ line_ $ if_ true_ [ line_ (var_ "c" .= 2) ]+          , line_ . return_ $ var_ "a" .+ var_ "b" .+ var_ "c"+          ]+    res <- fullyValidate expr+    res === Failure (FoundDynamic () (MkIdent () "c" []) :| [])++prop_scope_2 :: Property+prop_scope_2 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" [p_ "a", p_ "b"]+          [ line_ (var_ "c" .= 0)+          , line_ $ if_ true_ [ line_ (var_ "c" .= 2) ]+          , line_ . return_ $ var_ "a" .+ var_ "b" .+ var_ "c"+          ]+    res <- fullyValidate expr+    annotateShow res+    assert $ has _Success res++prop_scope_3 :: Property+prop_scope_3 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" [p_ "a", p_ "b"]+          [ line_ . return_ $ var_ "a" .+ var_ "b" .+ var_ "c" ]+    res <- fullyValidate expr+    annotateShow res+    res === Failure (NotInScope (MkIdent () "c" []) :| [])++prop_scope_4 :: Property+prop_scope_4 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" [p_ "a", p_ "b"]+          [ line_ $ def_ "f" [] [ line_ $ def_ "g" [] [ line_ pass_ ] ]+          , line_ $ call_ (var_ "g") []+          ]+    res <- fullyValidate expr+    res === Failure (NotInScope (MkIdent () "g" []) :| [])++prop_scope_5 :: Property+prop_scope_5 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" [p_ "a"]+          [ line_ $ def_ "f" [k_ "b" (var_ "c")] [ line_ pass_ ]+          ]+    res <- fullyValidate expr+    annotateShow res+    res === Failure (NotInScope (MkIdent () "c" []) :| [])++prop_scope_6 :: Property+prop_scope_6 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" []+          [ line_ $+              if_ true_ [ line_ (var_ "x" .= 2) ] &+              else_ [ line_ pass_ ]+          , line_ $ var_ "x"+          ]+    res <- fullyValidate expr+    annotateShow res+    res === Failure (FoundDynamic () (MkIdent () "x" []) :| [])++prop_scope_7 :: Property+prop_scope_7 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" []+          [ line_ $+              if_ true_ [ line_ pass_ ] &+              else_ [ line_ (var_ "x" .= 3) ]+          , line_ $ var_ "x"+          ]+    res <- fullyValidate expr+    annotateShow res+    res === Failure (FoundDynamic () (MkIdent () "x" []) :| [])++prop_scope_8 :: Property+prop_scope_8 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" []+          [ line_ $+              if_ true_ [ line_ pass_ ] &+              else_ [ line_ (var_ "x" .= 3) ]+          , line_ (var_ "x" .= 1)+          , line_ $ var_ "x"+          ]+    res <- fullyValidate expr+    annotateShow res+    (res $> ()) === Success ()++prop_scope_9 :: Property+prop_scope_9 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" []+          [ line_ $ for_ ("x" `in_` [ list_ [li_ $ int_ 1] ]) [ line_ pass_ ]+          , line_ $ var_ "x"+          ]+    res <- fullyValidate expr+    annotateShow res+    res === Failure (FoundDynamic () (MkIdent () "x" []) :| [])++prop_scope_10 :: Property+prop_scope_10 =+  withTests 1 . property $ do+    let+      expr =+        _Fundef #+        def_ "test" []+          [ line_ $ for_ ("x" `in_` [ list_ [li_ $ int_ 1] ]) [ line_ $ var_ "x" ]+          ]+    res <- fullyValidate expr+    annotateShow res+    (res $> ()) === Success ()++prop_scope_11 :: Property+prop_scope_11 =+  withTests 1 . property $ do+    let+      st =+        _Fundef #+        def_ "test" []+          [ line_ ("x" .= 2)+          , line_ $ for_ ("x" `in_` [ list_ [li_ $ int_ 1] ]) [ line_ pass_ ]+          ]+    res <- fullyValidate st+    annotateShow res+    res === Failure (BadShadowing (MkIdent () "x" [Space]) :| [])
+ test/Syntax.hs view
@@ -0,0 +1,180 @@+{-# language OverloadedStrings, TemplateHaskell #-}+{-# language DataKinds #-}+module Syntax (syntaxTests) where++import Hedgehog++import Control.Lens.Iso (from)+import Control.Lens.Getter ((^.))+import Control.Lens.Review ((#))+import Control.Monad (void)++import Language.Python.DSL+import Language.Python.Optics+import Language.Python.Parse (parseModule, parseStatement, parseExpr)+import Language.Python.Render (showStatement, showExpr)+import Language.Python.Syntax.CommaSep+import Language.Python.Syntax.Expr+import Language.Python.Syntax.Punctuation+import Language.Python.Syntax.Statement+import Language.Python.Syntax.Strings+import Language.Python.Syntax.Whitespace++import Helpers+  ( shouldBeParseSuccess, shouldBeFailure, shouldBeSuccess+  , syntaxValidateExpr, syntaxValidateStatement, syntaxValidateModule+  )++syntaxTests :: Group+syntaxTests = $$discover++prop_syntax_1 :: Property+prop_syntax_1 =+  withTests 1 . property $ do+    let+      e =+        -- lambda *: None+        Lambda ()+          [Space]+          (CommaSepMany (UnnamedStarParam () []) (MkComma []) CommaSepNone)+          (MkColon [Space])+          (None () [])+    res <- syntaxValidateExpr e+    shouldBeFailure res++prop_syntax_2 :: Property+prop_syntax_2 =+  withTests 1 . property $ do+    let+      i = replicate 4 Space ^. from indentWhitespaces+      e :: Statement '[] ()+      e =+        CompoundStatement .+        Fundef () []+          (Indents mempty ())+          Nothing+          (pure Space)+            "test"+            [] CommaSepNone [] Nothing .+          SuiteMany () (MkColon []) Nothing LF $+          Block []+            (SmallStatement (Indents [i] ()) $+             MkSmallStatement (Pass () []) [] Nothing Nothing Nothing)+            [Right . SmallStatement (Indents [i] ()) $+             MkSmallStatement (Pass () []) [] Nothing Nothing Nothing]+    res <- shouldBeParseSuccess parseStatement (showStatement e)+    res' <- shouldBeParseSuccess parseStatement (showStatement res)+    void res === void res'++prop_syntax_3 :: Property+prop_syntax_3 =+  withTests 1 . property $ do+    let+      s = "@a\ndef a():\n pass\n @a\n class a: return "+    e <- shouldBeParseSuccess parseModule s+    shouldBeFailure =<< syntaxValidateModule (() <$ e)++prop_syntax_4 :: Property+prop_syntax_4 =+  withTests 1 . property $ do+    let+      e :: Expr '[] ()+      e =+        String () . pure $+        StringLiteral ()+          Nothing+        ShortString SingleQuote+        [Char_lit '\\', Char_lit 'u']+        []+    res <- shouldBeParseSuccess parseExpr (showExpr e)+    res' <- shouldBeParseSuccess parseExpr (showExpr res)+    void res === void res'++prop_syntax_5 :: Property+prop_syntax_5 =+  withTests 1 . property $ do+    let+      e :: Expr '[] ()+      e =+        String () . pure $+        StringLiteral ()+          Nothing+        ShortString SingleQuote+        [Char_lit '\\', Char_lit 'x']+        []+    res <- shouldBeParseSuccess parseExpr (showExpr e)+    res' <- shouldBeParseSuccess parseExpr (showExpr res)+    void res === void res'++prop_syntax_6 :: Property+prop_syntax_6 =+  withTests 1 . property $ do+    let s= "async def a():\n class a(await None):\n  pass"+    e <- shouldBeParseSuccess parseModule s+    void . shouldBeSuccess =<< syntaxValidateModule (() <$ e)++prop_syntax_7 :: Property+prop_syntax_7 =+  withTests 1 . property $ do+    let+      s = "def a(b): global b"+    e <- shouldBeParseSuccess parseModule s+    shouldBeFailure =<< syntaxValidateModule (() <$ e)++prop_syntax_8 :: Property+prop_syntax_8 =+  withTests 1 . property $ do+    let+      s = "def a(*): pass"+    e <- shouldBeParseSuccess parseModule s+    shouldBeFailure =<< syntaxValidateModule (() <$ e)++prop_syntax_9 :: Property+prop_syntax_9 =+  withTests 1 . property $ do+    let+      s = "def a(*, b=None): pass"+    e <- shouldBeParseSuccess parseModule s+    void . shouldBeSuccess =<< syntaxValidateModule (() <$ e)++prop_syntax_10 :: Property+prop_syntax_10 =+  withTests 1 . property $ do+    let e = _Fundef # def_ "a" [s_ "b", s_ "c"] [line_ pass_]+    void . shouldBeFailure =<< syntaxValidateStatement e++prop_syntax_11 :: Property+prop_syntax_11 =+  withTests 1 . property $ do+    let e = lambda_ [s_ "a", s_ "b"] (var_ "a")+    void . shouldBeFailure =<< syntaxValidateExpr e++prop_syntax_12 :: Property+prop_syntax_12 =+  withTests 1 . property $ do+    let e = _Fundef # def_ "a" [star_, s_ "b"] [line_ pass_]+    void . shouldBeFailure =<< syntaxValidateStatement e++prop_syntax_13 :: Property+prop_syntax_13 =+  withTests 1 . property $ do+    let e = lambda_ [star_, s_ "a"] (var_ "b")+    void . shouldBeFailure =<< syntaxValidateExpr e++prop_syntax_14 :: Property+prop_syntax_14 =+  withTests 1 . property $ do+    let e = _Fundef # def_ "a" [star_, k_ "b" none_, s_ "c"] [line_ pass_]+    void . shouldBeFailure =<< syntaxValidateStatement e++prop_syntax_15 :: Property+prop_syntax_15 =+  withTests 1 . property $ do+    let e = lambda_ [star_, k_ "a" none_, s_ "b"] (var_ "c")+    void . shouldBeFailure =<< syntaxValidateExpr e++prop_syntax_16 :: Property+prop_syntax_16 =+  withTests 1 . property $ do+    let e = lambda_ [star_, star_, k_ "a" none_] (var_ "c")+    void . shouldBeFailure =<< syntaxValidateExpr e
+ test/files/ansible.py view
@@ -0,0 +1,939 @@+# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com>+# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>+#+# This file is part of Ansible+#+# Ansible is free software: you can redistribute it and/or modify+# it under the terms of the GNU General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# Ansible is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU General Public License+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.++# Make coding more python3-ish+from __future__ import (absolute_import, division, print_function)+__metaclass__ = type++import ast+import base64+import datetime+import imp+import json+import os+import shlex+import zipfile+import random+import re+from io import BytesIO++from ansible.release import __version__, __author__+from ansible import constants as C+from ansible.errors import AnsibleError+from ansible.module_utils._text import to_bytes, to_text, to_native+from ansible.plugins.loader import module_utils_loader, ps_module_utils_loader+from ansible.plugins.shell.powershell import async_watchdog, async_wrapper, become_wrapper, leaf_exec, exec_wrapper+# Must import strategy and use write_locks from there+# If we import write_locks directly then we end up binding a+# variable to the object and then it never gets updated.+from ansible.executor import action_write_locks++try:+    from __main__ import display+except ImportError:+    from ansible.utils.display import Display+    display = Display()+++REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"+REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""+REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""+REPLACER_WINDOWS = b"# POWERSHELL_COMMON"+REPLACER_JSONARGS = b"<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"+REPLACER_SELINUX = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"++# We could end up writing out parameters with unicode characters so we need to+# specify an encoding for the python source file+ENCODING_STRING = u'# -*- coding: utf-8 -*-'+b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'++# module_common is relative to module_utils, so fix the path+_MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')++# ******************************************************************************++ANSIBALLZ_TEMPLATE = u'''%(shebang)s+%(coding)s+ANSIBALLZ_WRAPPER = True # For test-module script to tell this is a ANSIBALLZ_WRAPPER+# This code is part of Ansible, but is an independent component.+# The code in this particular templatable string, and this templatable string+# only, is BSD licensed.  Modules which end up using this snippet, which is+# dynamically combined together by Ansible still belong to the author of the+# module, and they may assign their own license to the complete work.+#+# Copyright (c), James Cammarata, 2016+# Copyright (c), Toshio Kuratomi, 2016+#+# Redistribution and use in source and binary forms, with or without modification,+# are permitted provided that the following conditions are met:+#+#    * Redistributions of source code must retain the above copyright+#      notice, this list of conditions and the following disclaimer.+#    * Redistributions in binary form must reproduce the above copyright notice,+#      this list of conditions and the following disclaimer in the documentation+#      and/or other materials provided with the distribution.+#+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+import os+import os.path+import sys+import __main__++# For some distros and python versions we pick up this script in the temporary+# directory.  This leads to problems when the ansible module masks a python+# library that another import needs.  We have not figured out what about the+# specific distros and python versions causes this to behave differently.+#+# Tested distros:+# Fedora23 with python3.4  Works+# Ubuntu15.10 with python2.7  Works+# Ubuntu15.10 with python3.4  Fails without this+# Ubuntu16.04.1 with python3.5  Fails without this+# To test on another platform:+# * use the copy module (since this shadows the stdlib copy module)+# * Turn off pipelining+# * Make sure that the destination file does not exist+# * ansible ubuntu16-test -m copy -a 'src=/etc/motd dest=/var/tmp/m'+# This will traceback in shutil.  Looking at the complete traceback will show+# that shutil is importing copy which finds the ansible module instead of the+# stdlib module+scriptdir = None+try:+    scriptdir = os.path.dirname(os.path.realpath(__main__.__file__))+except (AttributeError, OSError):+    # Some platforms don't set __file__ when reading from stdin+    # OSX raises OSError if using abspath() in a directory we don't have+    # permission to read (realpath calls abspath)+    pass+if scriptdir is not None:+    sys.path = [p for p in sys.path if p != scriptdir]++import base64+import shutil+import zipfile+import tempfile+import subprocess++if sys.version_info < (3,):+    bytes = str+    PY3 = False+else:+    unicode = str+    PY3 = True+try:+    # Python-2.6++    from io import BytesIO as IOStream+except ImportError:+    # Python < 2.6+    from StringIO import StringIO as IOStream++ZIPDATA = """%(zipdata)s"""++def invoke_module(module, modlib_path, json_params):+    pythonpath = os.environ.get('PYTHONPATH')+    if pythonpath:+        os.environ['PYTHONPATH'] = ':'.join((modlib_path, pythonpath))+    else:+        os.environ['PYTHONPATH'] = modlib_path++    p = subprocess.Popen([%(interpreter)s, module], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)+    (stdout, stderr) = p.communicate(json_params)++    if not isinstance(stderr, (bytes, unicode)):+        stderr = stderr.read()+    if not isinstance(stdout, (bytes, unicode)):+        stdout = stdout.read()+    if PY3:+        sys.stderr.buffer.write(stderr)+        sys.stdout.buffer.write(stdout)+    else:+        sys.stderr.write(stderr)+        sys.stdout.write(stdout)+    return p.returncode++def debug(command, zipped_mod, json_params):+    # The code here normally doesn't run.  It's only used for debugging on the+    # remote machine.+    #+    # The subcommands in this function make it easier to debug ansiballz+    # modules.  Here's the basic steps:+    #+    # Run ansible with the environment variable: ANSIBLE_KEEP_REMOTE_FILES=1 and -vvv+    # to save the module file remotely::+    #   $ ANSIBLE_KEEP_REMOTE_FILES=1 ansible host1 -m ping -a 'data=october' -vvv+    #+    # Part of the verbose output will tell you where on the remote machine the+    # module was written to::+    #   [...]+    #   <host1> SSH: EXEC ssh -C -q -o ControlMaster=auto -o ControlPersist=60s -o KbdInteractiveAuthentication=no -o+    #   PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey -o PasswordAuthentication=no -o ConnectTimeout=10 -o+    #   ControlPath=/home/badger/.ansible/cp/ansible-ssh-%%h-%%p-%%r -tt rhel7 '/bin/sh -c '"'"'LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8+    #   LC_MESSAGES=en_US.UTF-8 /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping'"'"''+    #   [...]+    #+    # Login to the remote machine and run the module file via from the previous+    # step with the explode subcommand to extract the module payload into+    # source files::+    #   $ ssh host1+    #   $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping explode+    #   Module expanded into:+    #   /home/badger/.ansible/tmp/ansible-tmp-1461173408.08-279692652635227/ansible+    #+    # You can now edit the source files to instrument the code or experiment with+    # different parameter values.  When you're ready to run the code you've modified+    # (instead of the code from the actual zipped module), use the execute subcommand like this::+    #   $ /usr/bin/python /home/badger/.ansible/tmp/ansible-tmp-1461173013.93-9076457629738/ping execute++    # Okay to use __file__ here because we're running from a kept file+    basedir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'debug_dir')+    args_path = os.path.join(basedir, 'args')+    script_path = os.path.join(basedir, 'ansible_module_%(ansible_module)s.py')++    if command == 'explode':+        # transform the ZIPDATA into an exploded directory of code and then+        # print the path to the code.  This is an easy way for people to look+        # at the code on the remote machine for debugging it in that+        # environment+        z = zipfile.ZipFile(zipped_mod)+        for filename in z.namelist():+            if filename.startswith('/'):+                raise Exception('Something wrong with this module zip file: should not contain absolute paths')++            dest_filename = os.path.join(basedir, filename)+            if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename):+                os.makedirs(dest_filename)+            else:+                directory = os.path.dirname(dest_filename)+                if not os.path.exists(directory):+                    os.makedirs(directory)+                f = open(dest_filename, 'wb')+                f.write(z.read(filename))+                f.close()++        # write the args file+        f = open(args_path, 'wb')+        f.write(json_params)+        f.close()++        print('Module expanded into:')+        print('%%s' %% basedir)+        exitcode = 0++    elif command == 'execute':+        # Execute the exploded code instead of executing the module from the+        # embedded ZIPDATA.  This allows people to easily run their modified+        # code on the remote machine to see how changes will affect it.+        # This differs slightly from default Ansible execution of Python modules+        # as it passes the arguments to the module via a file instead of stdin.++        # Set pythonpath to the debug dir+        pythonpath = os.environ.get('PYTHONPATH')+        if pythonpath:+            os.environ['PYTHONPATH'] = ':'.join((basedir, pythonpath))+        else:+            os.environ['PYTHONPATH'] = basedir++        p = subprocess.Popen([%(interpreter)s, script_path, args_path],+                env=os.environ, shell=False, stdout=subprocess.PIPE,+                stderr=subprocess.PIPE, stdin=subprocess.PIPE)+        (stdout, stderr) = p.communicate()++        if not isinstance(stderr, (bytes, unicode)):+            stderr = stderr.read()+        if not isinstance(stdout, (bytes, unicode)):+            stdout = stdout.read()+        if PY3:+            sys.stderr.buffer.write(stderr)+            sys.stdout.buffer.write(stdout)+        else:+            sys.stderr.write(stderr)+            sys.stdout.write(stdout)+        return p.returncode++    elif command == 'excommunicate':+        # This attempts to run the module in-process (by importing a main+        # function and then calling it).  It is not the way ansible generally+        # invokes the module so it won't work in every case.  It is here to+        # aid certain debuggers which work better when the code doesn't change+        # from one process to another but there may be problems that occur+        # when using this that are only artifacts of how we're invoking here,+        # not actual bugs (as they don't affect the real way that we invoke+        # ansible modules)++        # stub the args and python path+        sys.argv = ['%(ansible_module)s', args_path]+        sys.path.insert(0, basedir)++        from ansible_module_%(ansible_module)s import main+        main()+        print('WARNING: Module returned to wrapper instead of exiting')+        sys.exit(1)+    else:+        print('WARNING: Unknown debug command.  Doing nothing.')+        exitcode = 0++    return exitcode++if __name__ == '__main__':+    #+    # See comments in the debug() method for information on debugging+    #++    ANSIBALLZ_PARAMS = %(params)s+    if PY3:+        ANSIBALLZ_PARAMS = ANSIBALLZ_PARAMS.encode('utf-8')+    try:+        # There's a race condition with the controller removing the+        # remote_tmpdir and this module executing under async.  So we cannot+        # store this in remote_tmpdir (use system tempdir instead)+        temp_path = tempfile.mkdtemp(prefix='ansible_')++        zipped_mod = os.path.join(temp_path, 'ansible_modlib.zip')+        modlib = open(zipped_mod, 'wb')+        modlib.write(base64.b64decode(ZIPDATA))+        modlib.close()++        if len(sys.argv) == 2:+            exitcode = debug(sys.argv[1], zipped_mod, ANSIBALLZ_PARAMS)+        else:+            z = zipfile.ZipFile(zipped_mod, mode='r')+            module = os.path.join(temp_path, 'ansible_module_%(ansible_module)s.py')+            f = open(module, 'wb')+            f.write(z.read('ansible_module_%(ansible_module)s.py'))+            f.close()++            # When installed via setuptools (including python setup.py install),+            # ansible may be installed with an easy-install.pth file.  That file+            # may load the system-wide install of ansible rather than the one in+            # the module.  sitecustomize is the only way to override that setting.+            z = zipfile.ZipFile(zipped_mod, mode='a')++            # py3: zipped_mod will be text, py2: it's bytes.  Need bytes at the end+            sitecustomize = u'import sys\\nsys.path.insert(0,"%%s")\\n' %%  zipped_mod+            sitecustomize = sitecustomize.encode('utf-8')+            # Use a ZipInfo to work around zipfile limitation on hosts with+            # clocks set to a pre-1980 year (for instance, Raspberry Pi)+            zinfo = zipfile.ZipInfo()+            zinfo.filename = 'sitecustomize.py'+            zinfo.date_time = ( %(year)i, %(month)i, %(day)i, %(hour)i, %(minute)i, %(second)i)+            z.writestr(zinfo, sitecustomize)+            z.close()++            exitcode = invoke_module(module, zipped_mod, ANSIBALLZ_PARAMS)+    finally:+        try:+            shutil.rmtree(temp_path)+        except (NameError, OSError):+            # tempdir creation probably failed+            pass+    sys.exit(exitcode)+'''+++def _strip_comments(source):+    # Strip comments and blank lines from the wrapper+    buf = []+    for line in source.splitlines():+        l = line.strip()+        if not l or l.startswith(u'#'):+            continue+        buf.append(line)+    return u'\n'.join(buf)+++if C.DEFAULT_KEEP_REMOTE_FILES:+    # Keep comments when KEEP_REMOTE_FILES is set.  That way users will see+    # the comments with some nice usage instructions+    ACTIVE_ANSIBALLZ_TEMPLATE = ANSIBALLZ_TEMPLATE+else:+    # ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size+    ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE)+++class ModuleDepFinder(ast.NodeVisitor):+    # Caveats:+    # This code currently does not handle:+    # * relative imports from py2.6+ from . import urls+    IMPORT_PREFIX_SIZE = len('ansible.module_utils.')+    def __init__(self, *args, **kwargs):+        """+        Walk the ast tree for the python module.++        Save submodule[.submoduleN][.identifier] into self.submodules++        self.submodules will end up with tuples like:+          - ('basic',)+          - ('urls', 'fetch_url')+          - ('database', 'postgres')+          - ('database', 'postgres', 'quote')++        It's up to calling code to determine whether the final element of the+        dotted strings are module names or something else (function, class, or+        variable names)+        """+        super(ModuleDepFinder, self).__init__(*args, **kwargs)+        self.submodules = set()++    def visit_Import(self, node):+        # import ansible.module_utils.MODLIB[.MODLIBn] [as asname]+        for alias in (a for a in node.names if a.name.startswith('ansible.module_utils.')):+            py_mod = alias.name[self.IMPORT_PREFIX_SIZE:]+            py_mod = tuple(py_mod.split('.'))+            self.submodules.add(py_mod)+        self.generic_visit(node)++    def visit_ImportFrom(self, node):+        # Specialcase: six is a special case because of its+        # import logic+        if node.names[0].name == '_six':+            self.submodules.add(('_six',))+        elif node.module.startswith('ansible.module_utils'):+            where_from = node.module[self.IMPORT_PREFIX_SIZE:]+            if where_from:+                # from ansible.module_utils.MODULE1[.MODULEn] import IDENTIFIER [as asname]+                # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [as asname]+                # from ansible.module_utils.MODULE1[.MODULEn] import MODULEn+1 [,IDENTIFIER] [as asname]+                py_mod = tuple(where_from.split('.'))+                for alias in node.names:+                    self.submodules.add(py_mod + (alias.name,))+            else:+                # from ansible.module_utils import MODLIB [,MODLIB2] [as asname]+                for alias in node.names:+                    self.submodules.add((alias.name,))+        self.generic_visit(node)+++def _slurp(path):+    if not os.path.exists(path):+        raise AnsibleError("imported module support code does not exist at %s" % os.path.abspath(path))+    fd = open(path, 'rb')+    data = fd.read()+    fd.close()+    return data+++def _get_shebang(interpreter, task_vars, templar, args=tuple()):+    """+    Note not stellar API:+       Returns None instead of always returning a shebang line.  Doing it this+       way allows the caller to decide to use the shebang it read from the+       file rather than trust that we reformatted what they already have+       correctly.+    """+    interpreter_config = u'ansible_%s_interpreter' % os.path.basename(interpreter).strip()++    if interpreter_config not in task_vars:+        return (None, interpreter)++    interpreter = templar.template(task_vars[interpreter_config].strip())+    shebang = u'#!' + interpreter++    if args:+        shebang = shebang + u' ' + u' '.join(args)++    return (shebang, interpreter)+++def recursive_finder(name, data, py_module_names, py_module_cache, zf):+    """+    Using ModuleDepFinder, make sure we have all of the module_utils files that+    the module its module_utils files needs.+    """+    # Parse the module and find the imports of ansible.module_utils+    tree = ast.parse(data)+    finder = ModuleDepFinder()+    finder.visit(tree)++    #+    # Determine what imports that we've found are modules (vs class, function.+    # variable names) for packages+    #++    normalized_modules = set()+    # Loop through the imports that we've found to normalize them+    # Exclude paths that match with paths we've already processed+    # (Have to exclude them a second time once the paths are processed)++    module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)]+    module_utils_paths.append(_MODULE_UTILS_PATH)+    for py_module_name in finder.submodules.difference(py_module_names):+        module_info = None++        if py_module_name[0] == 'six':+            # Special case the python six library because it messes up the+            # import process in an incompatible way+            module_info = imp.find_module('six', module_utils_paths)+            py_module_name = ('six',)+            idx = 0+        elif py_module_name[0] == '_six':+            # Special case the python six library because it messes up the+            # import process in an incompatible way+            module_info = imp.find_module('_six', [os.path.join(p, 'six') for p in module_utils_paths])+            py_module_name = ('six', '_six')+            idx = 0+        else:+            # Check whether either the last or the second to last identifier is+            # a module name+            for idx in (1, 2):+                if len(py_module_name) < idx:+                    break+                try:+                    module_info = imp.find_module(py_module_name[-idx],+                                                  [os.path.join(p, *py_module_name[:-idx]) for p in module_utils_paths])+                    break+                except ImportError:+                    continue++        # Could not find the module.  Construct a helpful error message.+        if module_info is None:+            msg = ['Could not find imported module support code for %s.  Looked for' % (name,)]+            if idx == 2:+                msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2]))+            else:+                msg.append(py_module_name[-1])+            raise AnsibleError(' '.join(msg))++        # Found a byte compiled file rather than source.  We cannot send byte+        # compiled over the wire as the python version might be different.+        # imp.find_module seems to prefer to return source packages so we just+        # error out if imp.find_module returns byte compiled files (This is+        # fragile as it depends on undocumented imp.find_module behaviour)+        if module_info[2][2] not in (imp.PY_SOURCE, imp.PKG_DIRECTORY):+            msg = ['Could not find python source for imported module support code for %s.  Looked for' % name]+            if idx == 2:+                msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2]))+            else:+                msg.append(py_module_name[-1])+            raise AnsibleError(' '.join(msg))++        if idx == 2:+            # We've determined that the last portion was an identifier and+            # thus, not part of the module name+            py_module_name = py_module_name[:-1]++        # If not already processed then we've got work to do+        if py_module_name not in py_module_names:+            # If not in the cache, then read the file into the cache+            # We already have a file handle for the module open so it makes+            # sense to read it now+            if py_module_name not in py_module_cache:+                if module_info[2][2] == imp.PKG_DIRECTORY:+                    # Read the __init__.py instead of the module file as this is+                    # a python package+                    normalized_name = py_module_name + ('__init__',)+                    normalized_path = os.path.join(os.path.join(module_info[1], '__init__.py'))+                    normalized_data = _slurp(normalized_path)+                else:+                    normalized_name = py_module_name+                    normalized_path = module_info[1]+                    normalized_data = module_info[0].read()+                    module_info[0].close()++                py_module_cache[normalized_name] = (normalized_data, normalized_path)+                normalized_modules.add(normalized_name)++            # Make sure that all the packages that this module is a part of+            # are also added+            for i in range(1, len(py_module_name)):+                py_pkg_name = py_module_name[:-i] + ('__init__',)+                if py_pkg_name not in py_module_names:+                    pkg_dir_info = imp.find_module(py_pkg_name[-1],+                                                   [os.path.join(p, *py_pkg_name[:-1]) for p in module_utils_paths])+                    normalized_modules.add(py_pkg_name)+                    py_module_cache[py_pkg_name] = (_slurp(pkg_dir_info[1]), pkg_dir_info[1])++    #+    # iterate through all of the ansible.module_utils* imports that we haven't+    # already checked for new imports+    #++    # set of modules that we haven't added to the zipfile+    unprocessed_py_module_names = normalized_modules.difference(py_module_names)++    for py_module_name in unprocessed_py_module_names:+        py_module_path = os.path.join(*py_module_name)+        py_module_file_name = '%s.py' % py_module_path++        zf.writestr(os.path.join("ansible/module_utils",+                    py_module_file_name), py_module_cache[py_module_name][0])+        display.vvvvv("Using module_utils file %s" % py_module_cache[py_module_name][1])++    # Add the names of the files we're scheduling to examine in the loop to+    # py_module_names so that we don't re-examine them in the next pass+    # through recursive_finder()+    py_module_names.update(unprocessed_py_module_names)++    for py_module_file in unprocessed_py_module_names:+        recursive_finder(py_module_file, py_module_cache[py_module_file][0], py_module_names, py_module_cache, zf)+        # Save memory; the file won't have to be read again for this ansible module.+        del py_module_cache[py_module_file]+++def _is_binary(b_module_data):+    textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))+    start = b_module_data[:1024]+    return bool(start.translate(None, textchars))+++def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression, async_timeout, become,+                       become_method, become_user, become_password, become_flags, environment):+    """+    Given the source of the module, convert it to a Jinja2 template to insert+    module code and return whether it's a new or old style module.+    """+    module_substyle = module_style = 'old'++    # module_style is something important to calling code (ActionBase).  It+    # determines how arguments are formatted (json vs k=v) and whether+    # a separate arguments file needs to be sent over the wire.+    # module_substyle is extra information that's useful internally.  It tells+    # us what we have to look to substitute in the module files and whether+    # we're using module replacer or ansiballz to format the module itself.+    if _is_binary(b_module_data):+        module_substyle = module_style = 'binary'+    elif REPLACER in b_module_data:+        # Do REPLACER before from ansible.module_utils because we need make sure+        # we substitute "from ansible.module_utils basic" for REPLACER+        module_style = 'new'+        module_substyle = 'python'+        b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')+    elif b'from ansible.module_utils.' in b_module_data:+        module_style = 'new'+        module_substyle = 'python'+    elif REPLACER_WINDOWS in b_module_data:+        module_style = 'new'+        module_substyle = 'powershell'+        b_module_data = b_module_data.replace(REPLACER_WINDOWS, b'#Requires -Module Ansible.ModuleUtils.Legacy')+    elif re.search(b'#Requires -Module', b_module_data, re.IGNORECASE) \+            or re.search(b'#Requires -Version', b_module_data, re.IGNORECASE)\+            or re.search(b'#AnsibleRequires -OSVersion', b_module_data, re.IGNORECASE):+        module_style = 'new'+        module_substyle = 'powershell'+    elif REPLACER_JSONARGS in b_module_data:+        module_style = 'new'+        module_substyle = 'jsonargs'+    elif b'WANT_JSON' in b_module_data:+        module_substyle = module_style = 'non_native_want_json'++    shebang = None+    # Neither old-style, non_native_want_json nor binary modules should be modified+    # except for the shebang line (Done by modify_module)+    if module_style in ('old', 'non_native_want_json', 'binary'):+        return b_module_data, module_style, shebang++    output = BytesIO()+    py_module_names = set()++    if module_substyle == 'python':+        params = dict(ANSIBLE_MODULE_ARGS=module_args,)+        python_repred_params = repr(json.dumps(params))++        try:+            compression_method = getattr(zipfile, module_compression)+        except AttributeError:+            display.warning(u'Bad module compression string specified: %s.  Using ZIP_STORED (no compression)' % module_compression)+            compression_method = zipfile.ZIP_STORED++        lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ansiballz_cache')+        cached_module_filename = os.path.join(lookup_path, "%s-%s" % (module_name, module_compression))++        zipdata = None+        # Optimization -- don't lock if the module has already been cached+        if os.path.exists(cached_module_filename):+            display.debug('ANSIBALLZ: using cached module: %s' % cached_module_filename)+            zipdata = open(cached_module_filename, 'rb').read()+        else:+            if module_name in action_write_locks.action_write_locks:+                display.debug('ANSIBALLZ: Using lock for %s' % module_name)+                lock = action_write_locks.action_write_locks[module_name]+            else:+                # If the action plugin directly invokes the module (instead of+                # going through a strategy) then we don't have a cross-process+                # Lock specifically for this module.  Use the "unexpected+                # module" lock instead+                display.debug('ANSIBALLZ: Using generic lock for %s' % module_name)+                lock = action_write_locks.action_write_locks[None]++            display.debug('ANSIBALLZ: Acquiring lock')+            with lock:+                display.debug('ANSIBALLZ: Lock acquired: %s' % id(lock))+                # Check that no other process has created this while we were+                # waiting for the lock+                if not os.path.exists(cached_module_filename):+                    display.debug('ANSIBALLZ: Creating module')+                    # Create the module zip data+                    zipoutput = BytesIO()+                    zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)+                    # Note: If we need to import from release.py first,+                    # remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523+                    zf.writestr('ansible/__init__.py',+                                b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' ++                                to_bytes(__version__) + b'"\n__author__="' ++                                to_bytes(__author__) + b'"\n')+                    zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n')++                    zf.writestr('ansible_module_%s.py' % module_name, b_module_data)++                    py_module_cache = {('__init__',): (b'', '[builtin]')}+                    recursive_finder(module_name, b_module_data, py_module_names, py_module_cache, zf)+                    zf.close()+                    zipdata = base64.b64encode(zipoutput.getvalue())++                    # Write the assembled module to a temp file (write to temp+                    # so that no one looking for the file reads a partially+                    # written file)+                    if not os.path.exists(lookup_path):+                        # Note -- if we have a global function to setup, that would+                        # be a better place to run this+                        os.makedirs(lookup_path)+                    display.debug('ANSIBALLZ: Writing module')+                    with open(cached_module_filename + '-part', 'wb') as f:+                        f.write(zipdata)++                    # Rename the file into its final position in the cache so+                    # future users of this module can read it off the+                    # filesystem instead of constructing from scratch.+                    display.debug('ANSIBALLZ: Renaming module')+                    os.rename(cached_module_filename + '-part', cached_module_filename)+                    display.debug('ANSIBALLZ: Done creating module')++            if zipdata is None:+                display.debug('ANSIBALLZ: Reading module after lock')+                # Another process wrote the file while we were waiting for+                # the write lock.  Go ahead and read the data from disk+                # instead of re-creating it.+                try:+                    zipdata = open(cached_module_filename, 'rb').read()+                except IOError:+                    raise AnsibleError('A different worker process failed to create module file. '+                                       'Look at traceback for that process for debugging information.')+        zipdata = to_text(zipdata, errors='surrogate_or_strict')++        shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars, templar)+        if shebang is None:+            shebang = u'#!/usr/bin/python'++        # Enclose the parts of the interpreter in quotes because we're+        # substituting it into the template as a Python string+        interpreter_parts = interpreter.split(u' ')+        interpreter = u"'{0}'".format(u"', '".join(interpreter_parts))++        now = datetime.datetime.utcnow()+        output.write(to_bytes(ACTIVE_ANSIBALLZ_TEMPLATE % dict(+            zipdata=zipdata,+            ansible_module=module_name,+            params=python_repred_params,+            shebang=shebang,+            interpreter=interpreter,+            coding=ENCODING_STRING,+            year=now.year,+            month=now.month,+            day=now.day,+            hour=now.hour,+            minute=now.minute,+            second=now.second,+        )))+        b_module_data = output.getvalue()++    elif module_substyle == 'powershell':+        # Powershell/winrm don't actually make use of shebang so we can+        # safely set this here.  If we let the fallback code handle this+        # it can fail in the presence of the UTF8 BOM commonly added by+        # Windows text editors+        shebang = u'#!powershell'++        exec_manifest = dict(+            module_entry=to_text(base64.b64encode(b_module_data)),+            powershell_modules=dict(),+            module_args=module_args,+            actions=['exec'],+            environment=environment+        )++        exec_manifest['exec'] = to_text(base64.b64encode(to_bytes(leaf_exec)))++        if async_timeout > 0:+            exec_manifest["actions"].insert(0, 'async_watchdog')+            exec_manifest["async_watchdog"] = to_text(base64.b64encode(to_bytes(async_watchdog)))+            exec_manifest["actions"].insert(0, 'async_wrapper')+            exec_manifest["async_wrapper"] = to_text(base64.b64encode(to_bytes(async_wrapper)))+            exec_manifest["async_jid"] = str(random.randint(0, 999999999999))+            exec_manifest["async_timeout_sec"] = async_timeout++        if become and become_method == 'runas':+            exec_manifest["actions"].insert(0, 'become')+            exec_manifest["become_user"] = become_user+            exec_manifest["become_password"] = become_password+            exec_manifest['become_flags'] = become_flags+            exec_manifest["become"] = to_text(base64.b64encode(to_bytes(become_wrapper)))++        lines = b_module_data.split(b'\n')+        module_names = set()+        become_required = False+        min_os_version = None+        min_ps_version = None++        requires_module_list = re.compile(to_bytes(r'(?i)^#\s*requires\s+\-module(?:s?)\s*(Ansible\.ModuleUtils\..+)'))+        requires_ps_version = re.compile(to_bytes(r'(?i)^#requires\s+\-version\s+([0-9]+(\.[0-9]+){0,3})$'))+        requires_os_version = re.compile(to_bytes(r'(?i)^#ansiblerequires\s+\-osversion\s+([0-9]+(\.[0-9]+){0,3})$'))+        requires_become = re.compile(to_bytes(r'(?i)^#ansiblerequires\s+\-become$'))++        for line in lines:+            module_util_line_match = requires_module_list.match(line)+            if module_util_line_match:+                module_names.add(module_util_line_match.group(1))++            requires_ps_version_match = requires_ps_version.match(line)+            if requires_ps_version_match:+                min_ps_version = to_text(requires_ps_version_match.group(1))+                # Powershell cannot cast a string of "1" to version, it must+                # have at least the major.minor for it to work so we append 0+                if requires_ps_version_match.group(2) is None:+                    min_ps_version = "%s.0" % min_ps_version++            requires_os_version_match = requires_os_version.match(line)+            if requires_os_version_match:+                min_os_version = to_text(requires_os_version_match.group(1))+                if requires_os_version_match.group(2) is None:+                    min_os_version = "%s.0" % min_os_version++            requires_become_match = requires_become.match(line)+            if requires_become_match:+                become_required = True++        for m in set(module_names):+            m = to_text(m).rstrip()  # tolerate windows line endings+            mu_path = ps_module_utils_loader.find_plugin(m, ".psm1")+            if not mu_path:+                raise AnsibleError('Could not find imported module support code for \'%s\'.' % m)+            exec_manifest["powershell_modules"][m] = to_text(+                base64.b64encode(+                    to_bytes(+                        _slurp(mu_path)+                    )+                )+            )++        exec_manifest['min_ps_version'] = min_ps_version+        exec_manifest['min_os_version'] = min_os_version+        if become_required and 'become' not in exec_manifest["actions"]:+            exec_manifest["actions"].insert(0, 'become')+            exec_manifest["become_user"] = "SYSTEM"+            exec_manifest["become_password"] = None+            exec_manifest['become_flags'] = None+            exec_manifest["become"] = to_text(base64.b64encode(to_bytes(become_wrapper)))++        # FUTURE: smuggle this back as a dict instead of serializing here; the connection plugin may need to modify it+        module_json = json.dumps(exec_manifest)++        b_module_data = exec_wrapper.replace(b"$json_raw = ''", b"$json_raw = @'\r\n%s\r\n'@" % to_bytes(module_json))++    elif module_substyle == 'jsonargs':+        module_args_json = to_bytes(json.dumps(module_args))++        # these strings could be included in a third-party module but+        # officially they were included in the 'basic' snippet for new-style+        # python modules (which has been replaced with something else in+        # ansiballz) If we remove them from jsonargs-style module replacer+        # then we can remove them everywhere.+        python_repred_args = to_bytes(repr(module_args_json))+        b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))+        b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args)+        b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))++        # The main event -- substitute the JSON args string into the module+        b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)++        facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict')+        b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility)++    return (b_module_data, module_style, shebang)+++def modify_module(module_name, module_path, module_args, templar, task_vars=None, module_compression='ZIP_STORED', async_timeout=0, become=False,+                  become_method=None, become_user=None, become_password=None, become_flags=None, environment=None):+    """+    Used to insert chunks of code into modules before transfer rather than+    doing regular python imports.  This allows for more efficient transfer in+    a non-bootstrapping scenario by not moving extra files over the wire and+    also takes care of embedding arguments in the transferred modules.++    This version is done in such a way that local imports can still be+    used in the module code, so IDEs don't have to be aware of what is going on.++    Example:++    from ansible.module_utils.basic import *++       ... will result in the insertion of basic.py into the module+       from the module_utils/ directory in the source tree.++    For powershell, this code effectively no-ops, as the exec wrapper requires access to a number of+    properties not available here.++    """+    task_vars = {} if task_vars is None else task_vars+    environment = {} if environment is None else environment++    with open(module_path, 'rb') as f:++        # read in the module source+        b_module_data = f.read()++    (b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, templar, module_compression,+                                                                async_timeout=async_timeout, become=become, become_method=become_method,+                                                                become_user=become_user, become_password=become_password, become_flags=become_flags,+                                                                environment=environment)++    if module_style == 'binary':+        return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))+    elif shebang is None:+        b_lines = b_module_data.split(b"\n", 1)+        if b_lines[0].startswith(b"#!"):+            b_shebang = b_lines[0].strip()+            # shlex.split on python-2.6 needs bytes.  On python-3.x it needs text+            args = shlex.split(to_native(b_shebang[2:], errors='surrogate_or_strict'))++            # _get_shebang() takes text strings+            args = [to_text(a, errors='surrogate_or_strict') for a in args]+            interpreter = args[0]+            b_new_shebang = to_bytes(_get_shebang(interpreter, task_vars, templar, args[1:])[0],+                                     errors='surrogate_or_strict', nonstring='passthru')++            if b_new_shebang:+                b_lines[0] = b_shebang = b_new_shebang++            if os.path.basename(interpreter).startswith(u'python'):+                b_lines.insert(1, b_ENCODING_STRING)++            shebang = to_text(b_shebang, nonstring='passthru', errors='surrogate_or_strict')+        else:+            # No shebang, assume a binary module?+            pass++        b_module_data = b"\n".join(b_lines)++    return (b_module_data, module_style, shebang)
+ test/files/asyncstatements.py view
@@ -0,0 +1,37 @@+@a+async def a():+    async with a as b:+        await f(1, 2, 3)++    async for x in y:+        await (lambda x: x)++async def a():+    pass++def a():+    pass++@a+def a():+    with a as b:+        pass++    for x in y:+        pass+++async def a():+    with a as b:+        pass++    async for x in y:+        pass++    await 3++async = 2+async+await = 2+await+print(async)
+ test/files/comments.py view
@@ -0,0 +1,5 @@+# blah+def boo():+    pass+       #bing+    #   bop
+ test/files/decorators.py view
@@ -0,0 +1,5 @@+@a++@a+def b():+  pass
+ test/files/dictcomp.py view
@@ -0,0 +1,4 @@+{() for a in ()} # set comprehension+{(): () for a in ()}+{' ' * min_indent for k in keys} # set comprehension+{k: ' ' * min_indent for k in keys}
+ test/files/django.py view
@@ -0,0 +1,688 @@+"""+Multi-part parsing for file uploads.++Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to+file upload handlers for processing.+"""+import base64+import binascii+import cgi+from urllib.parse import unquote++from django.conf import settings+from django.core.exceptions import (+    RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent,+)+from django.core.files.uploadhandler import (+    SkipFile, StopFutureHandlers, StopUpload,+)+from django.utils.datastructures import MultiValueDict+from django.utils.encoding import force_text+from django.utils.text import unescape_entities++__all__ = ('MultiPartParser', 'MultiPartParserError', 'InputStreamExhausted')+++class MultiPartParserError(Exception):+    pass+++class InputStreamExhausted(Exception):+    """+    No more reads are allowed from this device.+    """+    pass+++RAW = "raw"+FILE = "file"+FIELD = "field"+++class MultiPartParser:+    """+    A rfc2388 multipart/form-data parser.++    ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks+    and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``.+    """+    def __init__(self, META, input_data, upload_handlers, encoding=None):+        """+        Initialize the MultiPartParser object.++        :META:+            The standard ``META`` dictionary in Django request objects.+        :input_data:+            The raw post data, as a file-like object.+        :upload_handlers:+            A list of UploadHandler instances that perform operations on the+            uploaded data.+        :encoding:+            The encoding with which to treat the incoming data.+        """+        # Content-Type should contain multipart and the boundary information.+        content_type = META.get('CONTENT_TYPE', '')+        if not content_type.startswith('multipart/'):+            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)++        # Parse the header to get the boundary to split the parts.+        ctypes, opts = parse_header(content_type.encode('ascii'))+        boundary = opts.get('boundary')+        if not boundary or not cgi.valid_boundary(boundary):+            raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary.decode())++        # Content-Length should contain the length of the body we are about+        # to receive.+        try:+            content_length = int(META.get('CONTENT_LENGTH', 0))+        except (ValueError, TypeError):+            content_length = 0++        if content_length < 0:+            # This means we shouldn't continue...raise an error.+            raise MultiPartParserError("Invalid content length: %r" % content_length)++        if isinstance(boundary, str):+            boundary = boundary.encode('ascii')+        self._boundary = boundary+        self._input_data = input_data++        # For compatibility with low-level network APIs (with 32-bit integers),+        # the chunk size should be < 2^31, but still divisible by 4.+        possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size]+        self._chunk_size = min([2 ** 31 - 4] + possible_sizes)++        self._meta = META+        self._encoding = encoding or settings.DEFAULT_CHARSET+        self._content_length = content_length+        self._upload_handlers = upload_handlers++    def parse(self):+        """+        Parse the POST data and break it into a FILES MultiValueDict and a POST+        MultiValueDict.++        Return a tuple containing the POST and FILES dictionary, respectively.+        """+        from django.http import QueryDict++        encoding = self._encoding+        handlers = self._upload_handlers++        # HTTP spec says that Content-Length >= 0 is valid+        # handling content-length == 0 before continuing+        if self._content_length == 0:+            return QueryDict(encoding=self._encoding), MultiValueDict()++        # See if any of the handlers take care of the parsing.+        # This allows overriding everything if need be.+        for handler in handlers:+            result = handler.handle_raw_input(+                self._input_data,+                self._meta,+                self._content_length,+                self._boundary,+                encoding,+            )+            # Check to see if it was handled+            if result is not None:+                return result[0], result[1]++        # Create the data structures to be used later.+        self._post = QueryDict(mutable=True)+        self._files = MultiValueDict()++        # Instantiate the parser and stream:+        stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))++        # Whether or not to signal a file-completion at the beginning of the loop.+        old_field_name = None+        counters = [0] * len(handlers)++        # Number of bytes that have been read.+        num_bytes_read = 0+        # To count the number of keys in the request.+        num_post_keys = 0+        # To limit the amount of data read from the request.+        read_size = None++        try:+            for item_type, meta_data, field_stream in Parser(stream, self._boundary):+                if old_field_name:+                    # We run this at the beginning of the next loop+                    # since we cannot be sure a file is complete until+                    # we hit the next boundary/part of the multipart content.+                    self.handle_file_complete(old_field_name, counters)+                    old_field_name = None++                try:+                    disposition = meta_data['content-disposition'][1]+                    field_name = disposition['name'].strip()+                except (KeyError, IndexError, AttributeError):+                    continue++                transfer_encoding = meta_data.get('content-transfer-encoding')+                if transfer_encoding is not None:+                    transfer_encoding = transfer_encoding[0].strip()+                field_name = force_text(field_name, encoding, errors='replace')++                if item_type == FIELD:+                    # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.+                    num_post_keys += 1+                    if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and+                            settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys):+                        raise TooManyFieldsSent(+                            'The number of GET/POST parameters exceeded '+                            'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'+                        )++                    # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.+                    if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:+                        read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read++                    # This is a post field, we can just set it in the post+                    if transfer_encoding == 'base64':+                        raw_data = field_stream.read(size=read_size)+                        num_bytes_read += len(raw_data)+                        try:+                            data = base64.b64decode(raw_data)+                        except binascii.Error:+                            data = raw_data+                    else:+                        data = field_stream.read(size=read_size)+                        num_bytes_read += len(data)++                    # Add two here to make the check consistent with the+                    # x-www-form-urlencoded check that includes '&='.+                    num_bytes_read += len(field_name) + 2+                    if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and+                            num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):+                        raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')++                    self._post.appendlist(field_name, force_text(data, encoding, errors='replace'))+                elif item_type == FILE:+                    # This is a file, use the handler...+                    file_name = disposition.get('filename')+                    if file_name:+                        file_name = force_text(file_name, encoding, errors='replace')+                        file_name = self.IE_sanitize(unescape_entities(file_name))+                    if not file_name:+                        continue++                    content_type, content_type_extra = meta_data.get('content-type', ('', {}))+                    content_type = content_type.strip()+                    charset = content_type_extra.get('charset')++                    try:+                        content_length = int(meta_data.get('content-length')[0])+                    except (IndexError, TypeError, ValueError):+                        content_length = None++                    counters = [0] * len(handlers)+                    try:+                        for handler in handlers:+                            try:+                                handler.new_file(+                                    field_name, file_name, content_type,+                                    content_length, charset, content_type_extra,+                                )+                            except StopFutureHandlers:+                                break++                        for chunk in field_stream:+                            if transfer_encoding == 'base64':+                                # We only special-case base64 transfer encoding+                                # We should always decode base64 chunks by multiple of 4,+                                # ignoring whitespace.++                                stripped_chunk = b"".join(chunk.split())++                                remaining = len(stripped_chunk) % 4+                                while remaining != 0:+                                    over_chunk = field_stream.read(4 - remaining)+                                    stripped_chunk += b"".join(over_chunk.split())+                                    remaining = len(stripped_chunk) % 4++                                try:+                                    chunk = base64.b64decode(stripped_chunk)+                                except Exception as exc:+                                    # Since this is only a chunk, any error is an unfixable error.+                                    raise MultiPartParserError("Could not decode base64 data.") from exc++                            for i, handler in enumerate(handlers):+                                chunk_length = len(chunk)+                                chunk = handler.receive_data_chunk(chunk, counters[i])+                                counters[i] += chunk_length+                                if chunk is None:+                                    # Don't continue if the chunk received by+                                    # the handler is None.+                                    break++                    except SkipFile:+                        self._close_files()+                        # Just use up the rest of this file...+                        exhaust(field_stream)+                    else:+                        # Handle file upload completions on next iteration.+                        old_field_name = field_name+                else:+                    # If this is neither a FIELD or a FILE, just exhaust the stream.+                    exhaust(stream)+        except StopUpload as e:+            self._close_files()+            if not e.connection_reset:+                exhaust(self._input_data)+        else:+            # Make sure that the request data is all fed+            exhaust(self._input_data)++        # Signal that the upload has completed.+        # any() shortcircuits if a handler's upload_complete() returns a value.+        any(handler.upload_complete() for handler in handlers)+        self._post._mutable = False+        return self._post, self._files++    def handle_file_complete(self, old_field_name, counters):+        """+        Handle all the signaling that takes place when a file is complete.+        """+        for i, handler in enumerate(self._upload_handlers):+            file_obj = handler.file_complete(counters[i])+            if file_obj:+                # If it returns a file object, then set the files dict.+                self._files.appendlist(force_text(old_field_name, self._encoding, errors='replace'), file_obj)+                break++    def IE_sanitize(self, filename):+        """Cleanup filename from Internet Explorer full paths."""+        return filename and filename[filename.rfind("\\") + 1:].strip()++    def _close_files(self):+        # Free up all file handles.+        # FIXME: this currently assumes that upload handlers store the file as 'file'+        # We should document that... (Maybe add handler.free_file to complement new_file)+        for handler in self._upload_handlers:+            if hasattr(handler, 'file'):+                handler.file.close()+++class LazyStream:+    """+    The LazyStream wrapper allows one to get and "unget" bytes from a stream.++    Given a producer object (an iterator that yields bytestrings), the+    LazyStream object will support iteration, reading, and keeping a "look-back"+    variable in case you need to "unget" some bytes.+    """+    def __init__(self, producer, length=None):+        """+        Every LazyStream must have a producer when instantiated.++        A producer is an iterable that returns a string each time it+        is called.+        """+        self._producer = producer+        self._empty = False+        self._leftover = b''+        self.length = length+        self.position = 0+        self._remaining = length+        self._unget_history = []++    def tell(self):+        return self.position++    def read(self, size=None):+        def parts():+            remaining = self._remaining if size is None else size+            # do the whole thing in one shot if no limit was provided.+            if remaining is None:+                yield b''.join(self)+                return++            # otherwise do some bookkeeping to return exactly enough+            # of the stream and stashing any extra content we get from+            # the producer+            while remaining != 0:+                assert remaining > 0, 'remaining bytes to read should never go negative'++                try:+                    chunk = next(self)+                except StopIteration:+                    return+                else:+                    emitting = chunk[:remaining]+                    self.unget(chunk[remaining:])+                    remaining -= len(emitting)+                    yield emitting++        out = b''.join(parts())+        return out++    def __next__(self):+        """+        Used when the exact number of bytes to read is unimportant.++        Return whatever chunk is conveniently returned from the iterator.+        Useful to avoid unnecessary bookkeeping if performance is an issue.+        """+        if self._leftover:+            output = self._leftover+            self._leftover = b''+        else:+            output = next(self._producer)+            self._unget_history = []+        self.position += len(output)+        return output++    def close(self):+        """+        Used to invalidate/disable this lazy stream.++        Replace the producer with an empty list. Any leftover bytes that have+        already been read will still be reported upon read() and/or next().+        """+        self._producer = []++    def __iter__(self):+        return self++    def unget(self, bytes):+        """+        Place bytes back onto the front of the lazy stream.++        Future calls to read() will return those bytes first. The+        stream position and thus tell() will be rewound.+        """+        if not bytes:+            return+        self._update_unget_history(len(bytes))+        self.position -= len(bytes)+        self._leftover = bytes + self._leftover++    def _update_unget_history(self, num_bytes):+        """+        Update the unget history as a sanity check to see if we've pushed+        back the same number of bytes in one chunk. If we keep ungetting the+        same number of bytes many times (here, 50), we're mostly likely in an+        infinite loop of some sort. This is usually caused by a+        maliciously-malformed MIME request.+        """+        self._unget_history = [num_bytes] + self._unget_history[:49]+        number_equal = len([+            current_number for current_number in self._unget_history+            if current_number == num_bytes+        ])++        if number_equal > 40:+            raise SuspiciousMultipartForm(+                "The multipart parser got stuck, which shouldn't happen with"+                " normal uploaded files. Check for malicious upload activity;"+                " if there is none, report this to the Django developers."+            )+++class ChunkIter:+    """+    An iterable that will yield chunks of data. Given a file-like object as the+    constructor, yield chunks of read operations from that object.+    """+    def __init__(self, flo, chunk_size=64 * 1024):+        self.flo = flo+        self.chunk_size = chunk_size++    def __next__(self):+        try:+            data = self.flo.read(self.chunk_size)+        except InputStreamExhausted:+            raise StopIteration()+        if data:+            return data+        else:+            raise StopIteration()++    def __iter__(self):+        return self+++class InterBoundaryIter:+    """+    A Producer that will iterate over boundaries.+    """+    def __init__(self, stream, boundary):+        self._stream = stream+        self._boundary = boundary++    def __iter__(self):+        return self++    def __next__(self):+        try:+            return LazyStream(BoundaryIter(self._stream, self._boundary))+        except InputStreamExhausted:+            raise StopIteration()+++class BoundaryIter:+    """+    A Producer that is sensitive to boundaries.++    Will happily yield bytes until a boundary is found. Will yield the bytes+    before the boundary, throw away the boundary bytes themselves, and push the+    post-boundary bytes back on the stream.++    The future calls to next() after locating the boundary will raise a+    StopIteration exception.+    """++    def __init__(self, stream, boundary):+        self._stream = stream+        self._boundary = boundary+        self._done = False+        # rollback an additional six bytes because the format is like+        # this: CRLF<boundary>[--CRLF]+        self._rollback = len(boundary) + 6++        # Try to use mx fast string search if available. Otherwise+        # use Python find. Wrap the latter for consistency.+        unused_char = self._stream.read(1)+        if not unused_char:+            raise InputStreamExhausted()+        self._stream.unget(unused_char)++    def __iter__(self):+        return self++    def __next__(self):+        if self._done:+            raise StopIteration()++        stream = self._stream+        rollback = self._rollback++        bytes_read = 0+        chunks = []+        for bytes in stream:+            bytes_read += len(bytes)+            chunks.append(bytes)+            if bytes_read > rollback:+                break+            if not bytes:+                break+        else:+            self._done = True++        if not chunks:+            raise StopIteration()++        chunk = b''.join(chunks)+        boundary = self._find_boundary(chunk)++        if boundary:+            end, next = boundary+            stream.unget(chunk[next:])+            self._done = True+            return chunk[:end]+        else:+            # make sure we don't treat a partial boundary (and+            # its separators) as data+            if not chunk[:-rollback]:  # and len(chunk) >= (len(self._boundary) + 6):+                # There's nothing left, we should just return and mark as done.+                self._done = True+                return chunk+            else:+                stream.unget(chunk[-rollback:])+                return chunk[:-rollback]++    def _find_boundary(self, data):+        """+        Find a multipart boundary in data.++        Should no boundary exist in the data, return None. Otherwise, return+        a tuple containing the indices of the following:+         * the end of current encapsulation+         * the start of the next encapsulation+        """+        index = data.find(self._boundary)+        if index < 0:+            return None+        else:+            end = index+            next = index + len(self._boundary)+            # backup over CRLF+            last = max(0, end - 1)+            if data[last:last + 1] == b'\n':+                end -= 1+            last = max(0, end - 1)+            if data[last:last + 1] == b'\r':+                end -= 1+            return end, next+++def exhaust(stream_or_iterable):+    """Exhaust an iterator or stream."""+    try:+        iterator = iter(stream_or_iterable)+    except TypeError:+        iterator = ChunkIter(stream_or_iterable, 16384)++    for __ in iterator:+        pass+++def parse_boundary_stream(stream, max_header_size):+    """+    Parse one and exactly one stream that encapsulates a boundary.+    """+    # Stream at beginning of header, look for end of header+    # and parse it if found. The header must fit within one+    # chunk.+    chunk = stream.read(max_header_size)++    # 'find' returns the top of these four bytes, so we'll+    # need to munch them later to prevent them from polluting+    # the payload.+    header_end = chunk.find(b'\r\n\r\n')++    def _parse_header(line):+        main_value_pair, params = parse_header(line)+        try:+            name, value = main_value_pair.split(':', 1)+        except ValueError:+            raise ValueError("Invalid header: %r" % line)+        return name, (value, params)++    if header_end == -1:+        # we find no header, so we just mark this fact and pass on+        # the stream verbatim+        stream.unget(chunk)+        return (RAW, {}, stream)++    header = chunk[:header_end]++    # here we place any excess chunk back onto the stream, as+    # well as throwing away the CRLFCRLF bytes from above.+    stream.unget(chunk[header_end + 4:])++    TYPE = RAW+    outdict = {}++    # Eliminate blank lines+    for line in header.split(b'\r\n'):+        # This terminology ("main value" and "dictionary of+        # parameters") is from the Python docs.+        try:+            name, (value, params) = _parse_header(line)+        except ValueError:+            continue++        if name == 'content-disposition':+            TYPE = FIELD+            if params.get('filename'):+                TYPE = FILE++        outdict[name] = value, params++    if TYPE == RAW:+        stream.unget(chunk)++    return (TYPE, outdict, stream)+++class Parser:+    def __init__(self, stream, boundary):+        self._stream = stream+        self._separator = b'--' + boundary++    def __iter__(self):+        boundarystream = InterBoundaryIter(self._stream, self._separator)+        for sub_stream in boundarystream:+            # Iterate over each part+            yield parse_boundary_stream(sub_stream, 1024)+++def parse_header(line):+    """+    Parse the header into a key-value.++    Input (line): bytes, output: str for key/name, bytes for values which+    will be decoded later.+    """+    plist = _parse_header_params(b';' + line)+    key = plist.pop(0).lower().decode('ascii')+    pdict = {}+    for p in plist:+        i = p.find(b'=')+        if i >= 0:+            has_encoding = False+            name = p[:i].strip().lower().decode('ascii')+            if name.endswith('*'):+                # Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext")+                # http://tools.ietf.org/html/rfc2231#section-4+                name = name[:-1]+                if p.count(b"'") == 2:+                    has_encoding = True+            value = p[i + 1:].strip()+            if has_encoding:+                encoding, lang, value = value.split(b"'")+                value = unquote(value.decode(), encoding=encoding.decode())+            if len(value) >= 2 and value[:1] == value[-1:] == b'"':+                value = value[1:-1]+                value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"')+            pdict[name] = value+    return key, pdict+++def _parse_header_params(s):+    plist = []+    while s[:1] == b';':+        s = s[1:]+        end = s.find(b';')+        while end > 0 and s.count(b'"', 0, end) % 2:+            end = s.find(b';', end + 1)+        if end < 0:+            end = len(s)+        f = s[:end]+        plist.append(f.strip())+        s = s[end:]+    return plist
+ test/files/django2.py view
@@ -0,0 +1,2337 @@+import collections.abc+import copy+import datetime+import decimal+import operator+import uuid+import warnings+from base64 import b64decode, b64encode+from functools import partialmethod, total_ordering++from django import forms+from django.apps import apps+from django.conf import settings+from django.core import checks, exceptions, validators+# When the _meta object was formalized, this exception was moved to+# django.core.exceptions. It is retained here for backwards compatibility+# purposes.+from django.core.exceptions import FieldDoesNotExist  # NOQA+from django.db import connection, connections, router+from django.db.models.constants import LOOKUP_SEP+from django.db.models.query_utils import DeferredAttribute, RegisterLookupMixin+from django.utils import timezone+from django.utils.datastructures import DictWrapper+from django.utils.dateparse import (+    parse_date, parse_datetime, parse_duration, parse_time,+)+from django.utils.duration import duration_microseconds, duration_string+from django.utils.encoding import force_bytes, smart_text+from django.utils.functional import Promise, cached_property+from django.utils.ipv6 import clean_ipv6_address+from django.utils.itercompat import is_iterable+from django.utils.text import capfirst+from django.utils.translation import gettext_lazy as _++__all__ = [+    'AutoField', 'BLANK_CHOICE_DASH', 'BigAutoField', 'BigIntegerField',+    'BinaryField', 'BooleanField', 'CharField', 'CommaSeparatedIntegerField',+    'DateField', 'DateTimeField', 'DecimalField', 'DurationField',+    'EmailField', 'Empty', 'Field', 'FieldDoesNotExist', 'FilePathField',+    'FloatField', 'GenericIPAddressField', 'IPAddressField', 'IntegerField',+    'NOT_PROVIDED', 'NullBooleanField', 'PositiveIntegerField',+    'PositiveSmallIntegerField', 'SlugField', 'SmallIntegerField', 'TextField',+    'TimeField', 'URLField', 'UUIDField',+]+++class Empty:+    pass+++class NOT_PROVIDED:+    pass+++# The values to use for "blank" in SelectFields. Will be appended to the start+# of most "choices" lists.+BLANK_CHOICE_DASH = [("", "---------")]+++def _load_field(app_label, model_name, field_name):+    return apps.get_model(app_label, model_name)._meta.get_field(field_name)+++# A guide to Field parameters:+#+#   * name:      The name of the field specified in the model.+#   * attname:   The attribute to use on the model object. This is the same as+#                "name", except in the case of ForeignKeys, where "_id" is+#                appended.+#   * db_column: The db_column specified in the model (or None).+#   * column:    The database column for this field. This is the same as+#                "attname", except if db_column is specified.+#+# Code that introspects values, or does other dynamic things, should use+# attname. For example, this gets the primary key value of object "obj":+#+#     getattr(obj, opts.pk.attname)++def _empty(of_cls):+    new = Empty()+    new.__class__ = of_cls+    return new+++def return_None():+    return None+++@total_ordering+class Field(RegisterLookupMixin):+    """Base class for all field types"""++    # Designates whether empty strings fundamentally are allowed at the+    # database level.+    empty_strings_allowed = True+    empty_values = list(validators.EMPTY_VALUES)++    # These track each time a Field instance is created. Used to retain order.+    # The auto_creation_counter is used for fields that Django implicitly+    # creates, creation_counter is used for all user-specified fields.+    creation_counter = 0+    auto_creation_counter = -1+    default_validators = []  # Default set of validators+    default_error_messages = {+        'invalid_choice': _('Value %(value)r is not a valid choice.'),+        'null': _('This field cannot be null.'),+        'blank': _('This field cannot be blank.'),+        'unique': _('%(model_name)s with this %(field_label)s '+                    'already exists.'),+        # Translators: The 'lookup_type' is one of 'date', 'year' or 'month'.+        # Eg: "Title must be unique for pub_date year"+        'unique_for_date': _("%(field_label)s must be unique for "+                             "%(date_field_label)s %(lookup_type)s."),+    }+    system_check_deprecated_details = None+    system_check_removed_details = None++    # Field flags+    hidden = False++    many_to_many = None+    many_to_one = None+    one_to_many = None+    one_to_one = None+    related_model = None++    # Generic field type description, usually overridden by subclasses+    def _description(self):+        return _('Field of type: %(field_type)s') % {+            'field_type': self.__class__.__name__+        }+    description = property(_description)++    def __init__(self, verbose_name=None, name=None, primary_key=False,+                 max_length=None, unique=False, blank=False, null=False,+                 db_index=False, rel=None, default=NOT_PROVIDED, editable=True,+                 serialize=True, unique_for_date=None, unique_for_month=None,+                 unique_for_year=None, choices=None, help_text='', db_column=None,+                 db_tablespace=None, auto_created=False, validators=(),+                 error_messages=None):+        self.name = name+        self.verbose_name = verbose_name  # May be set by set_attributes_from_name+        self._verbose_name = verbose_name  # Store original for deconstruction+        self.primary_key = primary_key+        self.max_length, self._unique = max_length, unique+        self.blank, self.null = blank, null+        self.remote_field = rel+        self.is_relation = self.remote_field is not None+        self.default = default+        self.editable = editable+        self.serialize = serialize+        self.unique_for_date = unique_for_date+        self.unique_for_month = unique_for_month+        self.unique_for_year = unique_for_year+        if isinstance(choices, collections.abc.Iterator):+            choices = list(choices)+        self.choices = choices or []+        self.help_text = help_text+        self.db_index = db_index+        self.db_column = db_column+        self._db_tablespace = db_tablespace+        self.auto_created = auto_created++        # Adjust the appropriate creation counter, and save our local copy.+        if auto_created:+            self.creation_counter = Field.auto_creation_counter+            Field.auto_creation_counter -= 1+        else:+            self.creation_counter = Field.creation_counter+            Field.creation_counter += 1++        self._validators = list(validators)  # Store for deconstruction later++        messages = {}+        for c in reversed(self.__class__.__mro__):+            messages.update(getattr(c, 'default_error_messages', {}))+        messages.update(error_messages or {})+        self._error_messages = error_messages  # Store for deconstruction later+        self.error_messages = messages++    def __str__(self):+        """+        Return "app_label.model_label.field_name" for fields attached to+        models.+        """+        if not hasattr(self, 'model'):+            return super().__str__()+        model = self.model+        app = model._meta.app_label+        return '%s.%s.%s' % (app, model._meta.object_name, self.name)++    def __repr__(self):+        """Display the module, class, and name of the field."""+        path = '%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)+        name = getattr(self, 'name', None)+        if name is not None:+            return '<%s: %s>' % (path, name)+        return '<%s>' % path++    def check(self, **kwargs):+        return [+            *self._check_field_name(),+            *self._check_choices(),+            *self._check_db_index(),+            *self._check_null_allowed_for_primary_keys(),+            *self._check_backend_specific_checks(**kwargs),+            *self._check_validators(),+            *self._check_deprecation_details(),+        ]++    def _check_field_name(self):+        """+        Check if field name is valid, i.e. 1) does not end with an+        underscore, 2) does not contain "__" and 3) is not "pk".+        """+        if self.name.endswith('_'):+            return [+                checks.Error(+                    'Field names must not end with an underscore.',+                    obj=self,+                    id='fields.E001',+                )+            ]+        elif LOOKUP_SEP in self.name:+            return [+                checks.Error(+                    'Field names must not contain "%s".' % (LOOKUP_SEP,),+                    obj=self,+                    id='fields.E002',+                )+            ]+        elif self.name == 'pk':+            return [+                checks.Error(+                    "'pk' is a reserved word that cannot be used as a field name.",+                    obj=self,+                    id='fields.E003',+                )+            ]+        else:+            return []++    def _check_choices(self):+        if not self.choices:+            return []++        def is_value(value, accept_promise=True):+            return isinstance(value, (str, Promise) if accept_promise else str) or not is_iterable(value)++        if is_value(self.choices, accept_promise=False):+            return [+                checks.Error(+                    "'choices' must be an iterable (e.g., a list or tuple).",+                    obj=self,+                    id='fields.E004',+                )+            ]++        # Expect [group_name, [value, display]]+        for choices_group in self.choices:+            try:+                group_name, group_choices = choices_group+            except ValueError:+                # Containing non-pairs+                break+            try:+                if not all(+                    is_value(value) and is_value(human_name)+                    for value, human_name in group_choices+                ):+                    break+            except (TypeError, ValueError):+                # No groups, choices in the form [value, display]+                value, human_name = group_name, group_choices+                if not is_value(value) or not is_value(human_name):+                    break++            # Special case: choices=['ab']+            if isinstance(choices_group, str):+                break+        else:+            return []++        return [+            checks.Error(+                "'choices' must be an iterable containing "+                "(actual value, human readable name) tuples.",+                obj=self,+                id='fields.E005',+            )+        ]++    def _check_db_index(self):+        if self.db_index not in (None, True, False):+            return [+                checks.Error(+                    "'db_index' must be None, True or False.",+                    obj=self,+                    id='fields.E006',+                )+            ]+        else:+            return []++    def _check_null_allowed_for_primary_keys(self):+        if (self.primary_key and self.null and+                not connection.features.interprets_empty_strings_as_nulls):+            # We cannot reliably check this for backends like Oracle which+            # consider NULL and '' to be equal (and thus set up+            # character-based fields a little differently).+            return [+                checks.Error(+                    'Primary keys must not have null=True.',+                    hint=('Set null=False on the field, or '+                          'remove primary_key=True argument.'),+                    obj=self,+                    id='fields.E007',+                )+            ]+        else:+            return []++    def _check_backend_specific_checks(self, **kwargs):+        app_label = self.model._meta.app_label+        for db in connections:+            if router.allow_migrate(db, app_label, model_name=self.model._meta.model_name):+                return connections[db].validation.check_field(self, **kwargs)+        return []++    def _check_validators(self):+        errors = []+        for i, validator in enumerate(self.validators):+            if not callable(validator):+                errors.append(+                    checks.Error(+                        "All 'validators' must be callable.",+                        hint=(+                            "validators[{i}] ({repr}) isn't a function or "+                            "instance of a validator class.".format(+                                i=i, repr=repr(validator),+                            )+                        ),+                        obj=self,+                        id='fields.E008',+                    )+                )+        return errors++    def _check_deprecation_details(self):+        if self.system_check_removed_details is not None:+            return [+                checks.Error(+                    self.system_check_removed_details.get(+                        'msg',+                        '%s has been removed except for support in historical '+                        'migrations.' % self.__class__.__name__+                    ),+                    hint=self.system_check_removed_details.get('hint'),+                    obj=self,+                    id=self.system_check_removed_details.get('id', 'fields.EXXX'),+                )+            ]+        elif self.system_check_deprecated_details is not None:+            return [+                checks.Warning(+                    self.system_check_deprecated_details.get(+                        'msg',+                        '%s has been deprecated.' % self.__class__.__name__+                    ),+                    hint=self.system_check_deprecated_details.get('hint'),+                    obj=self,+                    id=self.system_check_deprecated_details.get('id', 'fields.WXXX'),+                )+            ]+        return []++    def get_col(self, alias, output_field=None):+        if output_field is None:+            output_field = self+        if alias != self.model._meta.db_table or output_field != self:+            from django.db.models.expressions import Col+            return Col(alias, self, output_field)+        else:+            return self.cached_col++    @cached_property+    def cached_col(self):+        from django.db.models.expressions import Col+        return Col(self.model._meta.db_table, self)++    def select_format(self, compiler, sql, params):+        """+        Custom format for select clauses. For example, GIS columns need to be+        selected as AsText(table.col) on MySQL as the table.col data can't be+        used by Django.+        """+        return sql, params++    def deconstruct(self):+        """+        Return enough information to recreate the field as a 4-tuple:++         * The name of the field on the model, if contribute_to_class() has+           been run.+         * The import path of the field, including the class:e.g.+           django.db.models.IntegerField This should be the most portable+           version, so less specific may be better.+         * A list of positional arguments.+         * A dict of keyword arguments.++        Note that the positional or keyword arguments must contain values of+        the following types (including inner values of collection types):++         * None, bool, str, int, float, complex, set, frozenset, list, tuple,+           dict+         * UUID+         * datetime.datetime (naive), datetime.date+         * top-level classes, top-level functions - will be referenced by their+           full import path+         * Storage instances - these have their own deconstruct() method++        This is because the values here must be serialized into a text format+        (possibly new Python code, possibly JSON) and these are the only types+        with encoding handlers defined.++        There's no need to return the exact way the field was instantiated this+        time, just ensure that the resulting field is the same - prefer keyword+        arguments over positional ones, and omit parameters with their default+        values.+        """+        # Short-form way of fetching all the default parameters+        keywords = {}+        possibles = {+            "verbose_name": None,+            "primary_key": False,+            "max_length": None,+            "unique": False,+            "blank": False,+            "null": False,+            "db_index": False,+            "default": NOT_PROVIDED,+            "editable": True,+            "serialize": True,+            "unique_for_date": None,+            "unique_for_month": None,+            "unique_for_year": None,+            "choices": [],+            "help_text": '',+            "db_column": None,+            "db_tablespace": None,+            "auto_created": False,+            "validators": [],+            "error_messages": None,+        }+        attr_overrides = {+            "unique": "_unique",+            "error_messages": "_error_messages",+            "validators": "_validators",+            "verbose_name": "_verbose_name",+            "db_tablespace": "_db_tablespace",+        }+        equals_comparison = {"choices", "validators"}+        for name, default in possibles.items():+            value = getattr(self, attr_overrides.get(name, name))+            # Unroll anything iterable for choices into a concrete list+            if name == "choices" and isinstance(value, collections.abc.Iterable):+                value = list(value)+            # Do correct kind of comparison+            if name in equals_comparison:+                if value != default:+                    keywords[name] = value+            else:+                if value is not default:+                    keywords[name] = value+        # Work out path - we shorten it for known Django core fields+        path = "%s.%s" % (self.__class__.__module__, self.__class__.__qualname__)+        if path.startswith("django.db.models.fields.related"):+            path = path.replace("django.db.models.fields.related", "django.db.models")+        if path.startswith("django.db.models.fields.files"):+            path = path.replace("django.db.models.fields.files", "django.db.models")+        if path.startswith("django.db.models.fields.proxy"):+            path = path.replace("django.db.models.fields.proxy", "django.db.models")+        if path.startswith("django.db.models.fields"):+            path = path.replace("django.db.models.fields", "django.db.models")+        # Return basic info - other fields should override this.+        return (self.name, path, [], keywords)++    def clone(self):+        """+        Uses deconstruct() to clone a new copy of this Field.+        Will not preserve any class attachments/attribute names.+        """+        name, path, args, kwargs = self.deconstruct()+        return self.__class__(*args, **kwargs)++    def __eq__(self, other):+        # Needed for @total_ordering+        if isinstance(other, Field):+            return self.creation_counter == other.creation_counter+        return NotImplemented++    def __lt__(self, other):+        # This is needed because bisect does not take a comparison function.+        if isinstance(other, Field):+            return self.creation_counter < other.creation_counter+        return NotImplemented++    def __hash__(self):+        return hash(self.creation_counter)++    def __deepcopy__(self, memodict):+        # We don't have to deepcopy very much here, since most things are not+        # intended to be altered after initial creation.+        obj = copy.copy(self)+        if self.remote_field:+            obj.remote_field = copy.copy(self.remote_field)+            if hasattr(self.remote_field, 'field') and self.remote_field.field is self:+                obj.remote_field.field = obj+        memodict[id(self)] = obj+        return obj++    def __copy__(self):+        # We need to avoid hitting __reduce__, so define this+        # slightly weird copy construct.+        obj = Empty()+        obj.__class__ = self.__class__+        obj.__dict__ = self.__dict__.copy()+        return obj++    def __reduce__(self):+        """+        Pickling should return the model._meta.fields instance of the field,+        not a new copy of that field. So, use the app registry to load the+        model and then the field back.+        """+        if not hasattr(self, 'model'):+            # Fields are sometimes used without attaching them to models (for+            # example in aggregation). In this case give back a plain field+            # instance. The code below will create a new empty instance of+            # class self.__class__, then update its dict with self.__dict__+            # values - so, this is very close to normal pickle.+            state = self.__dict__.copy()+            # The _get_default cached_property can't be pickled due to lambda+            # usage.+            state.pop('_get_default', None)+            return _empty, (self.__class__,), state+        return _load_field, (self.model._meta.app_label, self.model._meta.object_name,+                             self.name)++    def get_pk_value_on_save(self, instance):+        """+        Hook to generate new PK values on save. This method is called when+        saving instances with no primary key value set. If this method returns+        something else than None, then the returned value is used when saving+        the new instance.+        """+        if self.default:+            return self.get_default()+        return None++    def to_python(self, value):+        """+        Convert the input value into the expected Python data type, raising+        django.core.exceptions.ValidationError if the data can't be converted.+        Return the converted value. Subclasses should override this.+        """+        return value++    @cached_property+    def validators(self):+        """+        Some validators can't be created at field initialization time.+        This method provides a way to delay their creation until required.+        """+        return [*self.default_validators, *self._validators]++    def run_validators(self, value):+        if value in self.empty_values:+            return++        errors = []+        for v in self.validators:+            try:+                v(value)+            except exceptions.ValidationError as e:+                if hasattr(e, 'code') and e.code in self.error_messages:+                    e.message = self.error_messages[e.code]+                errors.extend(e.error_list)++        if errors:+            raise exceptions.ValidationError(errors)++    def validate(self, value, model_instance):+        """+        Validate value and raise ValidationError if necessary. Subclasses+        should override this to provide validation logic.+        """+        if not self.editable:+            # Skip validation for non-editable fields.+            return++        if self.choices and value not in self.empty_values:+            for option_key, option_value in self.choices:+                if isinstance(option_value, (list, tuple)):+                    # This is an optgroup, so look inside the group for+                    # options.+                    for optgroup_key, optgroup_value in option_value:+                        if value == optgroup_key:+                            return+                elif value == option_key:+                    return+            raise exceptions.ValidationError(+                self.error_messages['invalid_choice'],+                code='invalid_choice',+                params={'value': value},+            )++        if value is None and not self.null:+            raise exceptions.ValidationError(self.error_messages['null'], code='null')++        if not self.blank and value in self.empty_values:+            raise exceptions.ValidationError(self.error_messages['blank'], code='blank')++    def clean(self, value, model_instance):+        """+        Convert the value's type and run validation. Validation errors+        from to_python() and validate() are propagated. Return the correct+        value if no error is raised.+        """+        value = self.to_python(value)+        self.validate(value, model_instance)+        self.run_validators(value)+        return value++    def db_type_parameters(self, connection):+        return DictWrapper(self.__dict__, connection.ops.quote_name, 'qn_')++    def db_check(self, connection):+        """+        Return the database column check constraint for this field, for the+        provided connection. Works the same way as db_type() for the case that+        get_internal_type() does not map to a preexisting model field.+        """+        data = self.db_type_parameters(connection)+        try:+            return connection.data_type_check_constraints[self.get_internal_type()] % data+        except KeyError:+            return None++    def db_type(self, connection):+        """+        Return the database column data type for this field, for the provided+        connection.+        """+        # The default implementation of this method looks at the+        # backend-specific data_types dictionary, looking up the field by its+        # "internal type".+        #+        # A Field class can implement the get_internal_type() method to specify+        # which *preexisting* Django Field class it's most similar to -- i.e.,+        # a custom field might be represented by a TEXT column type, which is+        # the same as the TextField Django field type, which means the custom+        # field's get_internal_type() returns 'TextField'.+        #+        # But the limitation of the get_internal_type() / data_types approach+        # is that it cannot handle database column types that aren't already+        # mapped to one of the built-in Django field types. In this case, you+        # can implement db_type() instead of get_internal_type() to specify+        # exactly which wacky database column type you want to use.+        data = self.db_type_parameters(connection)+        try:+            return connection.data_types[self.get_internal_type()] % data+        except KeyError:+            return None++    def rel_db_type(self, connection):+        """+        Return the data type that a related field pointing to this field should+        use. For example, this method is called by ForeignKey and OneToOneField+        to determine its data type.+        """+        return self.db_type(connection)++    def cast_db_type(self, connection):+        """Return the data type to use in the Cast() function."""+        db_type = connection.ops.cast_data_types.get(self.get_internal_type())+        if db_type:+            return db_type % self.db_type_parameters(connection)+        return self.db_type(connection)++    def db_parameters(self, connection):+        """+        Extension of db_type(), providing a range of different return values+        (type, checks). This will look at db_type(), allowing custom model+        fields to override it.+        """+        type_string = self.db_type(connection)+        check_string = self.db_check(connection)+        return {+            "type": type_string,+            "check": check_string,+        }++    def db_type_suffix(self, connection):+        return connection.data_types_suffix.get(self.get_internal_type())++    def get_db_converters(self, connection):+        if hasattr(self, 'from_db_value'):+            return [self.from_db_value]+        return []++    @property+    def unique(self):+        return self._unique or self.primary_key++    @property+    def db_tablespace(self):+        return self._db_tablespace or settings.DEFAULT_INDEX_TABLESPACE++    def set_attributes_from_name(self, name):+        self.name = self.name or name+        self.attname, self.column = self.get_attname_column()+        self.concrete = self.column is not None+        if self.verbose_name is None and self.name:+            self.verbose_name = self.name.replace('_', ' ')++    def contribute_to_class(self, cls, name, private_only=False):+        """+        Register the field with the model class it belongs to.++        If private_only is True, create a separate instance of this field+        for every subclass of cls, even if cls is not an abstract model.+        """+        self.set_attributes_from_name(name)+        self.model = cls+        if private_only:+            cls._meta.add_field(self, private=True)+        else:+            cls._meta.add_field(self)+        if self.column:+            # Don't override classmethods with the descriptor. This means that+            # if you have a classmethod and a field with the same name, then+            # such fields can't be deferred (we don't have a check for this).+            if not getattr(cls, self.attname, None):+                setattr(cls, self.attname, DeferredAttribute(self.attname))+        if self.choices:+            setattr(cls, 'get_%s_display' % self.name,+                    partialmethod(cls._get_FIELD_display, field=self))++    def get_filter_kwargs_for_object(self, obj):+        """+        Return a dict that when passed as kwargs to self.model.filter(), would+        yield all instances having the same value for this field as obj has.+        """+        return {self.name: getattr(obj, self.attname)}++    def get_attname(self):+        return self.name++    def get_attname_column(self):+        attname = self.get_attname()+        column = self.db_column or attname+        return attname, column++    def get_internal_type(self):+        return self.__class__.__name__++    def pre_save(self, model_instance, add):+        """Return field's value just before saving."""+        return getattr(model_instance, self.attname)++    def get_prep_value(self, value):+        """Perform preliminary non-db specific value checks and conversions."""+        if isinstance(value, Promise):+            value = value._proxy____cast()+        return value++    def get_db_prep_value(self, value, connection, prepared=False):+        """+        Return field's value prepared for interacting with the database backend.++        Used by the default implementations of get_db_prep_save().+        """+        if not prepared:+            value = self.get_prep_value(value)+        return value++    def get_db_prep_save(self, value, connection):+        """Return field's value prepared for saving into a database."""+        return self.get_db_prep_value(value, connection=connection, prepared=False)++    def has_default(self):+        """Return a boolean of whether this field has a default value."""+        return self.default is not NOT_PROVIDED++    def get_default(self):+        """Return the default value for this field."""+        return self._get_default()++    @cached_property+    def _get_default(self):+        if self.has_default():+            if callable(self.default):+                return self.default+            return lambda: self.default++        if not self.empty_strings_allowed or self.null and not connection.features.interprets_empty_strings_as_nulls:+            return return_None+        return str  # return empty string++    def get_choices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH, limit_choices_to=None):+        """+        Return choices with a default blank choices included, for use+        as <select> choices for this field.+        """+        if self.choices:+            choices = list(self.choices)+            if include_blank:+                blank_defined = any(choice in ('', None) for choice, _ in self.flatchoices)+                if not blank_defined:+                    choices = blank_choice + choices+            return choices+        rel_model = self.remote_field.model+        limit_choices_to = limit_choices_to or self.get_limit_choices_to()+        choice_func = operator.attrgetter(+            self.remote_field.get_related_field().attname+            if hasattr(self.remote_field, 'get_related_field')+            else 'pk'+        )+        return (blank_choice if include_blank else []) + [+            (choice_func(x), smart_text(x))+            for x in rel_model._default_manager.complex_filter(limit_choices_to)+        ]++    def value_to_string(self, obj):+        """+        Return a string value of this field from the passed obj.+        This is used by the serialization framework.+        """+        return str(self.value_from_object(obj))++    def _get_flatchoices(self):+        """Flattened version of choices tuple."""+        flat = []+        for choice, value in self.choices:+            if isinstance(value, (list, tuple)):+                flat.extend(value)+            else:+                flat.append((choice, value))+        return flat+    flatchoices = property(_get_flatchoices)++    def save_form_data(self, instance, data):+        setattr(instance, self.name, data)++    def formfield(self, form_class=None, choices_form_class=None, **kwargs):+        """Return a django.forms.Field instance for this field."""+        defaults = {'required': not self.blank,+                    'label': capfirst(self.verbose_name),+                    'help_text': self.help_text}+        if self.has_default():+            if callable(self.default):+                defaults['initial'] = self.default+                defaults['show_hidden_initial'] = True+            else:+                defaults['initial'] = self.get_default()+        if self.choices:+            # Fields with choices get special treatment.+            include_blank = (self.blank or+                             not (self.has_default() or 'initial' in kwargs))+            defaults['choices'] = self.get_choices(include_blank=include_blank)+            defaults['coerce'] = self.to_python+            if self.null:+                defaults['empty_value'] = None+            if choices_form_class is not None:+                form_class = choices_form_class+            else:+                form_class = forms.TypedChoiceField+            # Many of the subclass-specific formfield arguments (min_value,+            # max_value) don't apply for choice fields, so be sure to only pass+            # the values that TypedChoiceField will understand.+            for k in list(kwargs):+                if k not in ('coerce', 'empty_value', 'choices', 'required',+                             'widget', 'label', 'initial', 'help_text',+                             'error_messages', 'show_hidden_initial', 'disabled'):+                    del kwargs[k]+        defaults.update(kwargs)+        if form_class is None:+            form_class = forms.CharField+        return form_class(**defaults)++    def value_from_object(self, obj):+        """Return the value of this field in the given model instance."""+        return getattr(obj, self.attname)+++class AutoField(Field):+    description = _("Integer")++    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value must be an integer."),+    }++    def __init__(self, *args, **kwargs):+        kwargs['blank'] = True+        super().__init__(*args, **kwargs)++    def check(self, **kwargs):+        return [+            *super().check(**kwargs),+            *self._check_primary_key(),+        ]++    def _check_primary_key(self):+        if not self.primary_key:+            return [+                checks.Error(+                    'AutoFields must set primary_key=True.',+                    obj=self,+                    id='fields.E100',+                ),+            ]+        else:+            return []++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        del kwargs['blank']+        kwargs['primary_key'] = True+        return name, path, args, kwargs++    def get_internal_type(self):+        return "AutoField"++    def to_python(self, value):+        if value is None:+            return value+        try:+            return int(value)+        except (TypeError, ValueError):+            raise exceptions.ValidationError(+                self.error_messages['invalid'],+                code='invalid',+                params={'value': value},+            )++    def rel_db_type(self, connection):+        return IntegerField().db_type(connection=connection)++    def validate(self, value, model_instance):+        pass++    def get_db_prep_value(self, value, connection, prepared=False):+        if not prepared:+            value = self.get_prep_value(value)+            value = connection.ops.validate_autopk_value(value)+        return value++    def get_prep_value(self, value):+        from django.db.models.expressions import OuterRef+        value = super().get_prep_value(value)+        if value is None or isinstance(value, OuterRef):+            return value+        return int(value)++    def contribute_to_class(self, cls, name, **kwargs):+        assert not cls._meta.auto_field, "Model %s can't have more than one AutoField." % cls._meta.label+        super().contribute_to_class(cls, name, **kwargs)+        cls._meta.auto_field = self++    def formfield(self, **kwargs):+        return None+++class BigAutoField(AutoField):+    description = _("Big (8 byte) integer")++    def get_internal_type(self):+        return "BigAutoField"++    def rel_db_type(self, connection):+        return BigIntegerField().db_type(connection=connection)+++class BooleanField(Field):+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value must be either True or False."),+        'invalid_nullable': _("'%(value)s' value must be either True, False, or None."),+    }+    description = _("Boolean (Either True or False)")++    def get_internal_type(self):+        return "BooleanField"++    def to_python(self, value):+        if self.null and value in self.empty_values:+            return None+        if value in (True, False):+            # 1/0 are equal to True/False. bool() converts former to latter.+            return bool(value)+        if value in ('t', 'True', '1'):+            return True+        if value in ('f', 'False', '0'):+            return False+        raise exceptions.ValidationError(+            self.error_messages['invalid_nullable' if self.null else 'invalid'],+            code='invalid',+            params={'value': value},+        )++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        if value is None:+            return None+        return self.to_python(value)++    def formfield(self, **kwargs):+        if self.choices:+            include_blank = not (self.has_default() or 'initial' in kwargs)+            defaults = {'choices': self.get_choices(include_blank=include_blank)}+        else:+            form_class = forms.NullBooleanField if self.null else forms.BooleanField+            # In HTML checkboxes, 'required' means "must be checked" which is+            # different from the choices case ("must select some value").+            # required=False allows unchecked checkboxes.+            defaults = {'form_class': form_class, 'required': False}+        return super().formfield(**{**defaults, **kwargs})+++class CharField(Field):+    description = _("String (up to %(max_length)s)")++    def __init__(self, *args, **kwargs):+        super().__init__(*args, **kwargs)+        self.validators.append(validators.MaxLengthValidator(self.max_length))++    def check(self, **kwargs):+        return [+            *super().check(**kwargs),+            *self._check_max_length_attribute(**kwargs),+        ]++    def _check_max_length_attribute(self, **kwargs):+        if self.max_length is None:+            return [+                checks.Error(+                    "CharFields must define a 'max_length' attribute.",+                    obj=self,+                    id='fields.E120',+                )+            ]+        elif (not isinstance(self.max_length, int) or isinstance(self.max_length, bool) or+                self.max_length <= 0):+            return [+                checks.Error(+                    "'max_length' must be a positive integer.",+                    obj=self,+                    id='fields.E121',+                )+            ]+        else:+            return []++    def cast_db_type(self, connection):+        if self.max_length is None:+            return connection.ops.cast_char_field_without_max_length+        return super().cast_db_type(connection)++    def get_internal_type(self):+        return "CharField"++    def to_python(self, value):+        if isinstance(value, str) or value is None:+            return value+        return str(value)++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        return self.to_python(value)++    def formfield(self, **kwargs):+        # Passing max_length to forms.CharField means that the value's length+        # will be validated twice. This is considered acceptable since we want+        # the value in the form field (to pass into widget for example).+        defaults = {'max_length': self.max_length}+        # TODO: Handle multiple backends with different feature flags.+        if self.null and not connection.features.interprets_empty_strings_as_nulls:+            defaults['empty_value'] = None+        defaults.update(kwargs)+        return super().formfield(**defaults)+++class CommaSeparatedIntegerField(CharField):+    default_validators = [validators.validate_comma_separated_integer_list]+    description = _("Comma-separated integers")+    system_check_removed_details = {+        'msg': (+            'CommaSeparatedIntegerField is removed except for support in '+            'historical migrations.'+        ),+        'hint': (+            'Use CharField(validators=[validate_comma_separated_integer_list]) '+            'instead.'+        ),+        'id': 'fields.E901',+    }+++class DateTimeCheckMixin:++    def check(self, **kwargs):+        return [+            *super().check(**kwargs),+            *self._check_mutually_exclusive_options(),+            *self._check_fix_default_value(),+        ]++    def _check_mutually_exclusive_options(self):+        # auto_now, auto_now_add, and default are mutually exclusive+        # options. The use of more than one of these options together+        # will trigger an Error+        mutually_exclusive_options = [self.auto_now_add, self.auto_now, self.has_default()]+        enabled_options = [option not in (None, False) for option in mutually_exclusive_options].count(True)+        if enabled_options > 1:+            return [+                checks.Error(+                    "The options auto_now, auto_now_add, and default "+                    "are mutually exclusive. Only one of these options "+                    "may be present.",+                    obj=self,+                    id='fields.E160',+                )+            ]+        else:+            return []++    def _check_fix_default_value(self):+        return []+++class DateField(DateTimeCheckMixin, Field):+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value has an invalid date format. It must be "+                     "in YYYY-MM-DD format."),+        'invalid_date': _("'%(value)s' value has the correct format (YYYY-MM-DD) "+                          "but it is an invalid date."),+    }+    description = _("Date (without time)")++    def __init__(self, verbose_name=None, name=None, auto_now=False,+                 auto_now_add=False, **kwargs):+        self.auto_now, self.auto_now_add = auto_now, auto_now_add+        if auto_now or auto_now_add:+            kwargs['editable'] = False+            kwargs['blank'] = True+        super().__init__(verbose_name, name, **kwargs)++    def _check_fix_default_value(self):+        """+        Warn that using an actual date or datetime value is probably wrong;+        it's only evaluated on server startup.+        """+        if not self.has_default():+            return []++        now = timezone.now()+        if not timezone.is_naive(now):+            now = timezone.make_naive(now, timezone.utc)+        value = self.default+        if isinstance(value, datetime.datetime):+            if not timezone.is_naive(value):+                value = timezone.make_naive(value, timezone.utc)+            value = value.date()+        elif isinstance(value, datetime.date):+            # Nothing to do, as dates don't have tz information+            pass+        else:+            # No explicit date / datetime value -- no checks necessary+            return []+        offset = datetime.timedelta(days=1)+        lower = (now - offset).date()+        upper = (now + offset).date()+        if lower <= value <= upper:+            return [+                checks.Warning(+                    'Fixed default value provided.',+                    hint='It seems you set a fixed date / time / datetime '+                         'value as default for this field. This may not be '+                         'what you want. If you want to have the current date '+                         'as default, use `django.utils.timezone.now`',+                    obj=self,+                    id='fields.W161',+                )+            ]++        return []++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if self.auto_now:+            kwargs['auto_now'] = True+        if self.auto_now_add:+            kwargs['auto_now_add'] = True+        if self.auto_now or self.auto_now_add:+            del kwargs['editable']+            del kwargs['blank']+        return name, path, args, kwargs++    def get_internal_type(self):+        return "DateField"++    def to_python(self, value):+        if value is None:+            return value+        if isinstance(value, datetime.datetime):+            if settings.USE_TZ and timezone.is_aware(value):+                # Convert aware datetimes to the default time zone+                # before casting them to dates (#17742).+                default_timezone = timezone.get_default_timezone()+                value = timezone.make_naive(value, default_timezone)+            return value.date()+        if isinstance(value, datetime.date):+            return value++        try:+            parsed = parse_date(value)+            if parsed is not None:+                return parsed+        except ValueError:+            raise exceptions.ValidationError(+                self.error_messages['invalid_date'],+                code='invalid_date',+                params={'value': value},+            )++        raise exceptions.ValidationError(+            self.error_messages['invalid'],+            code='invalid',+            params={'value': value},+        )++    def pre_save(self, model_instance, add):+        if self.auto_now or (self.auto_now_add and add):+            value = datetime.date.today()+            setattr(model_instance, self.attname, value)+            return value+        else:+            return super().pre_save(model_instance, add)++    def contribute_to_class(self, cls, name, **kwargs):+        super().contribute_to_class(cls, name, **kwargs)+        if not self.null:+            setattr(+                cls, 'get_next_by_%s' % self.name,+                partialmethod(cls._get_next_or_previous_by_FIELD, field=self, is_next=True)+            )+            setattr(+                cls, 'get_previous_by_%s' % self.name,+                partialmethod(cls._get_next_or_previous_by_FIELD, field=self, is_next=False)+            )++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        return self.to_python(value)++    def get_db_prep_value(self, value, connection, prepared=False):+        # Casts dates into the format expected by the backend+        if not prepared:+            value = self.get_prep_value(value)+        return connection.ops.adapt_datefield_value(value)++    def value_to_string(self, obj):+        val = self.value_from_object(obj)+        return '' if val is None else val.isoformat()++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.DateField,+            **kwargs,+        })+++class DateTimeField(DateField):+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value has an invalid format. It must be in "+                     "YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ] format."),+        'invalid_date': _("'%(value)s' value has the correct format "+                          "(YYYY-MM-DD) but it is an invalid date."),+        'invalid_datetime': _("'%(value)s' value has the correct format "+                              "(YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ]) "+                              "but it is an invalid date/time."),+    }+    description = _("Date (with time)")++    # __init__ is inherited from DateField++    def _check_fix_default_value(self):+        """+        Warn that using an actual date or datetime value is probably wrong;+        it's only evaluated on server startup.+        """+        if not self.has_default():+            return []++        now = timezone.now()+        if not timezone.is_naive(now):+            now = timezone.make_naive(now, timezone.utc)+        value = self.default+        if isinstance(value, datetime.datetime):+            second_offset = datetime.timedelta(seconds=10)+            lower = now - second_offset+            upper = now + second_offset+            if timezone.is_aware(value):+                value = timezone.make_naive(value, timezone.utc)+        elif isinstance(value, datetime.date):+            second_offset = datetime.timedelta(seconds=10)+            lower = now - second_offset+            lower = datetime.datetime(lower.year, lower.month, lower.day)+            upper = now + second_offset+            upper = datetime.datetime(upper.year, upper.month, upper.day)+            value = datetime.datetime(value.year, value.month, value.day)+        else:+            # No explicit date / datetime value -- no checks necessary+            return []+        if lower <= value <= upper:+            return [+                checks.Warning(+                    'Fixed default value provided.',+                    hint='It seems you set a fixed date / time / datetime '+                         'value as default for this field. This may not be '+                         'what you want. If you want to have the current date '+                         'as default, use `django.utils.timezone.now`',+                    obj=self,+                    id='fields.W161',+                )+            ]++        return []++    def get_internal_type(self):+        return "DateTimeField"++    def to_python(self, value):+        if value is None:+            return value+        if isinstance(value, datetime.datetime):+            return value+        if isinstance(value, datetime.date):+            value = datetime.datetime(value.year, value.month, value.day)+            if settings.USE_TZ:+                # For backwards compatibility, interpret naive datetimes in+                # local time. This won't work during DST change, but we can't+                # do much about it, so we let the exceptions percolate up the+                # call stack.+                warnings.warn("DateTimeField %s.%s received a naive datetime "+                              "(%s) while time zone support is active." %+                              (self.model.__name__, self.name, value),+                              RuntimeWarning)+                default_timezone = timezone.get_default_timezone()+                value = timezone.make_aware(value, default_timezone)+            return value++        try:+            parsed = parse_datetime(value)+            if parsed is not None:+                return parsed+        except ValueError:+            raise exceptions.ValidationError(+                self.error_messages['invalid_datetime'],+                code='invalid_datetime',+                params={'value': value},+            )++        try:+            parsed = parse_date(value)+            if parsed is not None:+                return datetime.datetime(parsed.year, parsed.month, parsed.day)+        except ValueError:+            raise exceptions.ValidationError(+                self.error_messages['invalid_date'],+                code='invalid_date',+                params={'value': value},+            )++        raise exceptions.ValidationError(+            self.error_messages['invalid'],+            code='invalid',+            params={'value': value},+        )++    def pre_save(self, model_instance, add):+        if self.auto_now or (self.auto_now_add and add):+            value = timezone.now()+            setattr(model_instance, self.attname, value)+            return value+        else:+            return super().pre_save(model_instance, add)++    # contribute_to_class is inherited from DateField, it registers+    # get_next_by_FOO and get_prev_by_FOO++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        value = self.to_python(value)+        if value is not None and settings.USE_TZ and timezone.is_naive(value):+            # For backwards compatibility, interpret naive datetimes in local+            # time. This won't work during DST change, but we can't do much+            # about it, so we let the exceptions percolate up the call stack.+            try:+                name = '%s.%s' % (self.model.__name__, self.name)+            except AttributeError:+                name = '(unbound)'+            warnings.warn("DateTimeField %s received a naive datetime (%s)"+                          " while time zone support is active." %+                          (name, value),+                          RuntimeWarning)+            default_timezone = timezone.get_default_timezone()+            value = timezone.make_aware(value, default_timezone)+        return value++    def get_db_prep_value(self, value, connection, prepared=False):+        # Casts datetimes into the format expected by the backend+        if not prepared:+            value = self.get_prep_value(value)+        return connection.ops.adapt_datetimefield_value(value)++    def value_to_string(self, obj):+        val = self.value_from_object(obj)+        return '' if val is None else val.isoformat()++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.DateTimeField,+            **kwargs,+        })+++class DecimalField(Field):+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value must be a decimal number."),+    }+    description = _("Decimal number")++    def __init__(self, verbose_name=None, name=None, max_digits=None,+                 decimal_places=None, **kwargs):+        self.max_digits, self.decimal_places = max_digits, decimal_places+        super().__init__(verbose_name, name, **kwargs)++    def check(self, **kwargs):+        errors = super().check(**kwargs)++        digits_errors = [+            *self._check_decimal_places(),+            *self._check_max_digits(),+        ]+        if not digits_errors:+            errors.extend(self._check_decimal_places_and_max_digits(**kwargs))+        else:+            errors.extend(digits_errors)+        return errors++    def _check_decimal_places(self):+        try:+            decimal_places = int(self.decimal_places)+            if decimal_places < 0:+                raise ValueError()+        except TypeError:+            return [+                checks.Error(+                    "DecimalFields must define a 'decimal_places' attribute.",+                    obj=self,+                    id='fields.E130',+                )+            ]+        except ValueError:+            return [+                checks.Error(+                    "'decimal_places' must be a non-negative integer.",+                    obj=self,+                    id='fields.E131',+                )+            ]+        else:+            return []++    def _check_max_digits(self):+        try:+            max_digits = int(self.max_digits)+            if max_digits <= 0:+                raise ValueError()+        except TypeError:+            return [+                checks.Error(+                    "DecimalFields must define a 'max_digits' attribute.",+                    obj=self,+                    id='fields.E132',+                )+            ]+        except ValueError:+            return [+                checks.Error(+                    "'max_digits' must be a positive integer.",+                    obj=self,+                    id='fields.E133',+                )+            ]+        else:+            return []++    def _check_decimal_places_and_max_digits(self, **kwargs):+        if int(self.decimal_places) > int(self.max_digits):+            return [+                checks.Error(+                    "'max_digits' must be greater or equal to 'decimal_places'.",+                    obj=self,+                    id='fields.E134',+                )+            ]+        return []++    @cached_property+    def validators(self):+        return super().validators + [+            validators.DecimalValidator(self.max_digits, self.decimal_places)+        ]++    @cached_property+    def context(self):+        return decimal.Context(prec=self.max_digits)++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if self.max_digits is not None:+            kwargs['max_digits'] = self.max_digits+        if self.decimal_places is not None:+            kwargs['decimal_places'] = self.decimal_places+        return name, path, args, kwargs++    def get_internal_type(self):+        return "DecimalField"++    def to_python(self, value):+        if value is None:+            return value+        if isinstance(value, float):+            return self.context.create_decimal_from_float(value)+        try:+            return decimal.Decimal(value)+        except decimal.InvalidOperation:+            raise exceptions.ValidationError(+                self.error_messages['invalid'],+                code='invalid',+                params={'value': value},+            )++    def get_db_prep_save(self, value, connection):+        return connection.ops.adapt_decimalfield_value(self.to_python(value), self.max_digits, self.decimal_places)++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        return self.to_python(value)++    def formfield(self, **kwargs):+        return super().formfield(**{+            'max_digits': self.max_digits,+            'decimal_places': self.decimal_places,+            'form_class': forms.DecimalField,+            **kwargs,+        })+++class DurationField(Field):+    """+    Store timedelta objects.++    Use interval on PostgreSQL, INTERVAL DAY TO SECOND on Oracle, and bigint+    of microseconds on other databases.+    """+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value has an invalid format. It must be in "+                     "[DD] [HH:[MM:]]ss[.uuuuuu] format.")+    }+    description = _("Duration")++    def get_internal_type(self):+        return "DurationField"++    def to_python(self, value):+        if value is None:+            return value+        if isinstance(value, datetime.timedelta):+            return value+        try:+            parsed = parse_duration(value)+        except ValueError:+            pass+        else:+            if parsed is not None:+                return parsed++        raise exceptions.ValidationError(+            self.error_messages['invalid'],+            code='invalid',+            params={'value': value},+        )++    def get_db_prep_value(self, value, connection, prepared=False):+        if connection.features.has_native_duration_field:+            return value+        if value is None:+            return None+        return duration_microseconds(value)++    def get_db_converters(self, connection):+        converters = []+        if not connection.features.has_native_duration_field:+            converters.append(connection.ops.convert_durationfield_value)+        return converters + super().get_db_converters(connection)++    def value_to_string(self, obj):+        val = self.value_from_object(obj)+        return '' if val is None else duration_string(val)++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.DurationField,+            **kwargs,+        })+++class EmailField(CharField):+    default_validators = [validators.validate_email]+    description = _("Email address")++    def __init__(self, *args, **kwargs):+        # max_length=254 to be compliant with RFCs 3696 and 5321+        kwargs.setdefault('max_length', 254)+        super().__init__(*args, **kwargs)++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        # We do not exclude max_length if it matches default as we want to change+        # the default in future.+        return name, path, args, kwargs++    def formfield(self, **kwargs):+        # As with CharField, this will cause email validation to be performed+        # twice.+        return super().formfield(**{+            'form_class': forms.EmailField,+            **kwargs,+        })+++class FilePathField(Field):+    description = _("File path")++    def __init__(self, verbose_name=None, name=None, path='', match=None,+                 recursive=False, allow_files=True, allow_folders=False, **kwargs):+        self.path, self.match, self.recursive = path, match, recursive+        self.allow_files, self.allow_folders = allow_files, allow_folders+        kwargs.setdefault('max_length', 100)+        super().__init__(verbose_name, name, **kwargs)++    def check(self, **kwargs):+        return [+            *super().check(**kwargs),+            *self._check_allowing_files_or_folders(**kwargs),+        ]++    def _check_allowing_files_or_folders(self, **kwargs):+        if not self.allow_files and not self.allow_folders:+            return [+                checks.Error(+                    "FilePathFields must have either 'allow_files' or 'allow_folders' set to True.",+                    obj=self,+                    id='fields.E140',+                )+            ]+        return []++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if self.path != '':+            kwargs['path'] = self.path+        if self.match is not None:+            kwargs['match'] = self.match+        if self.recursive is not False:+            kwargs['recursive'] = self.recursive+        if self.allow_files is not True:+            kwargs['allow_files'] = self.allow_files+        if self.allow_folders is not False:+            kwargs['allow_folders'] = self.allow_folders+        if kwargs.get("max_length") == 100:+            del kwargs["max_length"]+        return name, path, args, kwargs++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        if value is None:+            return None+        return str(value)++    def formfield(self, **kwargs):+        return super().formfield(**{+            'path': self.path,+            'match': self.match,+            'recursive': self.recursive,+            'form_class': forms.FilePathField,+            'allow_files': self.allow_files,+            'allow_folders': self.allow_folders,+            **kwargs,+        })++    def get_internal_type(self):+        return "FilePathField"+++class FloatField(Field):+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value must be a float."),+    }+    description = _("Floating point number")++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        if value is None:+            return None+        return float(value)++    def get_internal_type(self):+        return "FloatField"++    def to_python(self, value):+        if value is None:+            return value+        try:+            return float(value)+        except (TypeError, ValueError):+            raise exceptions.ValidationError(+                self.error_messages['invalid'],+                code='invalid',+                params={'value': value},+            )++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.FloatField,+            **kwargs,+        })+++class IntegerField(Field):+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value must be an integer."),+    }+    description = _("Integer")++    def check(self, **kwargs):+        return [+            *super().check(**kwargs),+            *self._check_max_length_warning(),+        ]++    def _check_max_length_warning(self):+        if self.max_length is not None:+            return [+                checks.Warning(+                    "'max_length' is ignored when used with IntegerField",+                    hint="Remove 'max_length' from field",+                    obj=self,+                    id='fields.W122',+                )+            ]+        return []++    @cached_property+    def validators(self):+        # These validators can't be added at field initialization time since+        # they're based on values retrieved from `connection`.+        validators_ = super().validators+        internal_type = self.get_internal_type()+        min_value, max_value = connection.ops.integer_field_range(internal_type)+        if (min_value is not None and not+            any(isinstance(validator, validators.MinValueValidator) and+                validator.limit_value >= min_value for validator in validators_)):+            validators_.append(validators.MinValueValidator(min_value))+        if (max_value is not None and not+            any(isinstance(validator, validators.MaxValueValidator) and+                validator.limit_value <= max_value for validator in validators_)):+            validators_.append(validators.MaxValueValidator(max_value))+        return validators_++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        if value is None:+            return None+        return int(value)++    def get_internal_type(self):+        return "IntegerField"++    def to_python(self, value):+        if value is None:+            return value+        try:+            return int(value)+        except (TypeError, ValueError):+            raise exceptions.ValidationError(+                self.error_messages['invalid'],+                code='invalid',+                params={'value': value},+            )++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.IntegerField,+            **kwargs,+        })+++class BigIntegerField(IntegerField):+    empty_strings_allowed = False+    description = _("Big (8 byte) integer")+    MAX_BIGINT = 9223372036854775807++    def get_internal_type(self):+        return "BigIntegerField"++    def formfield(self, **kwargs):+        return super().formfield(**{+            'min_value': -BigIntegerField.MAX_BIGINT - 1,+            'max_value': BigIntegerField.MAX_BIGINT,+            **kwargs,+        })+++class IPAddressField(Field):+    empty_strings_allowed = False+    description = _("IPv4 address")+    system_check_removed_details = {+        'msg': (+            'IPAddressField has been removed except for support in '+            'historical migrations.'+        ),+        'hint': 'Use GenericIPAddressField instead.',+        'id': 'fields.E900',+    }++    def __init__(self, *args, **kwargs):+        kwargs['max_length'] = 15+        super().__init__(*args, **kwargs)++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        del kwargs['max_length']+        return name, path, args, kwargs++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        if value is None:+            return None+        return str(value)++    def get_internal_type(self):+        return "IPAddressField"+++class GenericIPAddressField(Field):+    empty_strings_allowed = False+    description = _("IP address")+    default_error_messages = {}++    def __init__(self, verbose_name=None, name=None, protocol='both',+                 unpack_ipv4=False, *args, **kwargs):+        self.unpack_ipv4 = unpack_ipv4+        self.protocol = protocol+        self.default_validators, invalid_error_message = \+            validators.ip_address_validators(protocol, unpack_ipv4)+        self.default_error_messages['invalid'] = invalid_error_message+        kwargs['max_length'] = 39+        super().__init__(verbose_name, name, *args, **kwargs)++    def check(self, **kwargs):+        return [+            *super().check(**kwargs),+            *self._check_blank_and_null_values(**kwargs),+        ]++    def _check_blank_and_null_values(self, **kwargs):+        if not getattr(self, 'null', False) and getattr(self, 'blank', False):+            return [+                checks.Error(+                    'GenericIPAddressFields cannot have blank=True if null=False, '+                    'as blank values are stored as nulls.',+                    obj=self,+                    id='fields.E150',+                )+            ]+        return []++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if self.unpack_ipv4 is not False:+            kwargs['unpack_ipv4'] = self.unpack_ipv4+        if self.protocol != "both":+            kwargs['protocol'] = self.protocol+        if kwargs.get("max_length") == 39:+            del kwargs['max_length']+        return name, path, args, kwargs++    def get_internal_type(self):+        return "GenericIPAddressField"++    def to_python(self, value):+        if value is None:+            return None+        if not isinstance(value, str):+            value = str(value)+        value = value.strip()+        if ':' in value:+            return clean_ipv6_address(value, self.unpack_ipv4, self.error_messages['invalid'])+        return value++    def get_db_prep_value(self, value, connection, prepared=False):+        if not prepared:+            value = self.get_prep_value(value)+        return connection.ops.adapt_ipaddressfield_value(value)++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        if value is None:+            return None+        if value and ':' in value:+            try:+                return clean_ipv6_address(value, self.unpack_ipv4)+            except exceptions.ValidationError:+                pass+        return str(value)++    def formfield(self, **kwargs):+        return super().formfield(**{+            'protocol': self.protocol,+            'form_class': forms.GenericIPAddressField,+            **kwargs,+        })+++class NullBooleanField(BooleanField):+    default_error_messages = {+        'invalid': _("'%(value)s' value must be either None, True or False."),+        'invalid_nullable': _("'%(value)s' value must be either None, True or False."),+    }+    description = _("Boolean (Either True, False or None)")++    def __init__(self, *args, **kwargs):+        kwargs['null'] = True+        kwargs['blank'] = True+        super().__init__(*args, **kwargs)++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        del kwargs['null']+        del kwargs['blank']+        return name, path, args, kwargs++    def get_internal_type(self):+        return "NullBooleanField"+++class PositiveIntegerRelDbTypeMixin:++    def rel_db_type(self, connection):+        """+        Return the data type that a related field pointing to this field should+        use. In most cases, a foreign key pointing to a positive integer+        primary key will have an integer column data type but some databases+        (e.g. MySQL) have an unsigned integer type. In that case+        (related_fields_match_type=True), the primary key should return its+        db_type.+        """+        if connection.features.related_fields_match_type:+            return self.db_type(connection)+        else:+            return IntegerField().db_type(connection=connection)+++class PositiveIntegerField(PositiveIntegerRelDbTypeMixin, IntegerField):+    description = _("Positive integer")++    def get_internal_type(self):+        return "PositiveIntegerField"++    def formfield(self, **kwargs):+        return super().formfield(**{+            'min_value': 0,+            **kwargs,+        })+++class PositiveSmallIntegerField(PositiveIntegerRelDbTypeMixin, IntegerField):+    description = _("Positive small integer")++    def get_internal_type(self):+        return "PositiveSmallIntegerField"++    def formfield(self, **kwargs):+        return super().formfield(**{+            'min_value': 0,+            **kwargs,+        })+++class SlugField(CharField):+    default_validators = [validators.validate_slug]+    description = _("Slug (up to %(max_length)s)")++    def __init__(self, *args, max_length=50, db_index=True, allow_unicode=False, **kwargs):+        self.allow_unicode = allow_unicode+        if self.allow_unicode:+            self.default_validators = [validators.validate_unicode_slug]+        super().__init__(*args, max_length=max_length, db_index=db_index, **kwargs)++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if kwargs.get("max_length") == 50:+            del kwargs['max_length']+        if self.db_index is False:+            kwargs['db_index'] = False+        else:+            del kwargs['db_index']+        if self.allow_unicode is not False:+            kwargs['allow_unicode'] = self.allow_unicode+        return name, path, args, kwargs++    def get_internal_type(self):+        return "SlugField"++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.SlugField,+            'allow_unicode': self.allow_unicode,+            **kwargs,+        })+++class SmallIntegerField(IntegerField):+    description = _("Small integer")++    def get_internal_type(self):+        return "SmallIntegerField"+++class TextField(Field):+    description = _("Text")++    def get_internal_type(self):+        return "TextField"++    def to_python(self, value):+        if isinstance(value, str) or value is None:+            return value+        return str(value)++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        return self.to_python(value)++    def formfield(self, **kwargs):+        # Passing max_length to forms.CharField means that the value's length+        # will be validated twice. This is considered acceptable since we want+        # the value in the form field (to pass into widget for example).+        return super().formfield(**{+            'max_length': self.max_length,+            **({} if self.choices else {'widget': forms.Textarea}),+            **kwargs,+        })+++class TimeField(DateTimeCheckMixin, Field):+    empty_strings_allowed = False+    default_error_messages = {+        'invalid': _("'%(value)s' value has an invalid format. It must be in "+                     "HH:MM[:ss[.uuuuuu]] format."),+        'invalid_time': _("'%(value)s' value has the correct format "+                          "(HH:MM[:ss[.uuuuuu]]) but it is an invalid time."),+    }+    description = _("Time")++    def __init__(self, verbose_name=None, name=None, auto_now=False,+                 auto_now_add=False, **kwargs):+        self.auto_now, self.auto_now_add = auto_now, auto_now_add+        if auto_now or auto_now_add:+            kwargs['editable'] = False+            kwargs['blank'] = True+        super().__init__(verbose_name, name, **kwargs)++    def _check_fix_default_value(self):+        """+        Warn that using an actual date or datetime value is probably wrong;+        it's only evaluated on server startup.+        """+        if not self.has_default():+            return []++        now = timezone.now()+        if not timezone.is_naive(now):+            now = timezone.make_naive(now, timezone.utc)+        value = self.default+        if isinstance(value, datetime.datetime):+            second_offset = datetime.timedelta(seconds=10)+            lower = now - second_offset+            upper = now + second_offset+            if timezone.is_aware(value):+                value = timezone.make_naive(value, timezone.utc)+        elif isinstance(value, datetime.time):+            second_offset = datetime.timedelta(seconds=10)+            lower = now - second_offset+            upper = now + second_offset+            value = datetime.datetime.combine(now.date(), value)+            if timezone.is_aware(value):+                value = timezone.make_naive(value, timezone.utc).time()+        else:+            # No explicit time / datetime value -- no checks necessary+            return []+        if lower <= value <= upper:+            return [+                checks.Warning(+                    'Fixed default value provided.',+                    hint='It seems you set a fixed date / time / datetime '+                         'value as default for this field. This may not be '+                         'what you want. If you want to have the current date '+                         'as default, use `django.utils.timezone.now`',+                    obj=self,+                    id='fields.W161',+                )+            ]++        return []++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if self.auto_now is not False:+            kwargs["auto_now"] = self.auto_now+        if self.auto_now_add is not False:+            kwargs["auto_now_add"] = self.auto_now_add+        if self.auto_now or self.auto_now_add:+            del kwargs['blank']+            del kwargs['editable']+        return name, path, args, kwargs++    def get_internal_type(self):+        return "TimeField"++    def to_python(self, value):+        if value is None:+            return None+        if isinstance(value, datetime.time):+            return value+        if isinstance(value, datetime.datetime):+            # Not usually a good idea to pass in a datetime here (it loses+            # information), but this can be a side-effect of interacting with a+            # database backend (e.g. Oracle), so we'll be accommodating.+            return value.time()++        try:+            parsed = parse_time(value)+            if parsed is not None:+                return parsed+        except ValueError:+            raise exceptions.ValidationError(+                self.error_messages['invalid_time'],+                code='invalid_time',+                params={'value': value},+            )++        raise exceptions.ValidationError(+            self.error_messages['invalid'],+            code='invalid',+            params={'value': value},+        )++    def pre_save(self, model_instance, add):+        if self.auto_now or (self.auto_now_add and add):+            value = datetime.datetime.now().time()+            setattr(model_instance, self.attname, value)+            return value+        else:+            return super().pre_save(model_instance, add)++    def get_prep_value(self, value):+        value = super().get_prep_value(value)+        return self.to_python(value)++    def get_db_prep_value(self, value, connection, prepared=False):+        # Casts times into the format expected by the backend+        if not prepared:+            value = self.get_prep_value(value)+        return connection.ops.adapt_timefield_value(value)++    def value_to_string(self, obj):+        val = self.value_from_object(obj)+        return '' if val is None else val.isoformat()++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.TimeField,+            **kwargs,+        })+++class URLField(CharField):+    default_validators = [validators.URLValidator()]+    description = _("URL")++    def __init__(self, verbose_name=None, name=None, **kwargs):+        kwargs.setdefault('max_length', 200)+        super().__init__(verbose_name, name, **kwargs)++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if kwargs.get("max_length") == 200:+            del kwargs['max_length']+        return name, path, args, kwargs++    def formfield(self, **kwargs):+        # As with CharField, this will cause URL validation to be performed+        # twice.+        return super().formfield(**{+            'form_class': forms.URLField,+            **kwargs,+        })+++class BinaryField(Field):+    description = _("Raw binary data")+    empty_values = [None, b'']++    def __init__(self, *args, **kwargs):+        kwargs.setdefault('editable', False)+        super().__init__(*args, **kwargs)+        if self.max_length is not None:+            self.validators.append(validators.MaxLengthValidator(self.max_length))++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        if self.editable:+            kwargs['editable'] = True+        else:+            del kwargs['editable']+        return name, path, args, kwargs++    def get_internal_type(self):+        return "BinaryField"++    def get_placeholder(self, value, compiler, connection):+        return connection.ops.binary_placeholder_sql(value)++    def get_default(self):+        if self.has_default() and not callable(self.default):+            return self.default+        default = super().get_default()+        if default == '':+            return b''+        return default++    def get_db_prep_value(self, value, connection, prepared=False):+        value = super().get_db_prep_value(value, connection, prepared)+        if value is not None:+            return connection.Database.Binary(value)+        return value++    def value_to_string(self, obj):+        """Binary data is serialized as base64"""+        return b64encode(force_bytes(self.value_from_object(obj))).decode('ascii')++    def to_python(self, value):+        # If it's a string, it should be base64-encoded data+        if isinstance(value, str):+            return memoryview(b64decode(force_bytes(value)))+        return value+++class UUIDField(Field):+    default_error_messages = {+        'invalid': _("'%(value)s' is not a valid UUID."),+    }+    description = 'Universally unique identifier'+    empty_strings_allowed = False++    def __init__(self, verbose_name=None, **kwargs):+        kwargs['max_length'] = 32+        super().__init__(verbose_name, **kwargs)++    def deconstruct(self):+        name, path, args, kwargs = super().deconstruct()+        del kwargs['max_length']+        return name, path, args, kwargs++    def get_internal_type(self):+        return "UUIDField"++    def get_db_prep_value(self, value, connection, prepared=False):+        if value is None:+            return None+        if not isinstance(value, uuid.UUID):+            value = self.to_python(value)++        if connection.features.has_native_uuid_field:+            return value+        return value.hex++    def to_python(self, value):+        if value is not None and not isinstance(value, uuid.UUID):+            try:+                return uuid.UUID(value)+            except (AttributeError, ValueError):+                raise exceptions.ValidationError(+                    self.error_messages['invalid'],+                    code='invalid',+                    params={'value': value},+                )+        return value++    def formfield(self, **kwargs):+        return super().formfield(**{+            'form_class': forms.UUIDField,+            **kwargs,+        })
+ test/files/imaginary.py view
@@ -0,0 +1,13 @@+3.14j+10.j+10j+.001j+1e100j+3.14e-10j+03.14j+010.j+010j+0e0j+0.001j+01e100j+03.14e-10j
+ test/files/indent_optics_in.py view
@@ -0,0 +1,6 @@+def fact(x):+	ret = 1+	if x >= 1:+		for i in range(1,x+1):+			ret = ret * i+	return ret
+ test/files/indent_optics_in2.py view
@@ -0,0 +1,5 @@+def fib(x):+  if x < 2:+    return 1+  else:+    return fib(x-1) + fib(x-2)
+ test/files/indent_optics_out.py view
@@ -0,0 +1,6 @@+def fact(x):+    ret = 1+    if x >= 1:+        for i in range(1,x+1):+            ret = ret * i+    return ret
+ test/files/indent_optics_out2.py view
@@ -0,0 +1,5 @@+def fib(x):+    if x < 2:+        return 1+    else:+        return fib(x-1) + fib(x-2)
+ test/files/joblib.py view
@@ -0,0 +1,1098 @@+"""+This class is defined to override standard pickle functionality++The goals of it follow:+-Serialize lambdas and nested functions to compiled byte code+-Deal with main module correctly+-Deal with other non-serializable objects++It does not include an unpickler, as standard python unpickling suffices.++This module was extracted from the `cloud` package, developed by `PiCloud, Inc.+<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.++Copyright (c) 2012, Regents of the University of California.+Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.+All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions+are met:+    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.+    * Redistributions in binary form must reproduce the above copyright+      notice, this list of conditions and the following disclaimer in the+      documentation and/or other materials provided with the distribution.+    * Neither the name of the University of California, Berkeley nor the+      names of its contributors may be used to endorse or promote+      products derived from this software without specific prior written+      permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+"""+from __future__ import print_function++import dis+from functools import partial+import imp+import io+import itertools+import logging+import opcode+import operator+import pickle+import struct+import sys+import traceback+import types+import weakref+++# cloudpickle is meant for inter process communication: we expect all+# communicating processes to run the same Python version hence we favor+# communication speed over compatibility:+DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL+++if sys.version < '3':+    from pickle import Pickler+    try:+        from cStringIO import StringIO+    except ImportError:+        from StringIO import StringIO+    PY3 = False+else:+    types.ClassType = type+    from pickle import _Pickler as Pickler+    from io import BytesIO as StringIO+    PY3 = True+++def _make_cell_set_template_code():+    """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF++    Notes+    -----+    In Python 3, we could use an easier function:++    .. code-block:: python++       def f():+           cell = None++           def _stub(value):+               nonlocal cell+               cell = value++           return _stub++        _cell_set_template_code = f()++    This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is+    invalid syntax on Python 2. If we use this function we also don't need+    to do the weird freevars/cellvars swap below+    """+    def inner(value):+        lambda: cell  # make ``cell`` a closure so that we get a STORE_DEREF+        cell = value++    co = inner.__code__++    # NOTE: we are marking the cell variable as a free variable intentionally+    # so that we simulate an inner function instead of the outer function. This+    # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way.+    if not PY3:+        return types.CodeType(+            co.co_argcount,+            co.co_nlocals,+            co.co_stacksize,+            co.co_flags,+            co.co_code,+            co.co_consts,+            co.co_names,+            co.co_varnames,+            co.co_filename,+            co.co_name,+            co.co_firstlineno,+            co.co_lnotab,+            co.co_cellvars,  # this is the trickery+            (),+        )+    else:+        return types.CodeType(+            co.co_argcount,+            co.co_kwonlyargcount,+            co.co_nlocals,+            co.co_stacksize,+            co.co_flags,+            co.co_code,+            co.co_consts,+            co.co_names,+            co.co_varnames,+            co.co_filename,+            co.co_name,+            co.co_firstlineno,+            co.co_lnotab,+            co.co_cellvars,  # this is the trickery+            (),+        )+++_cell_set_template_code = _make_cell_set_template_code()+++def cell_set(cell, value):+    """Set the value of a closure cell.+    """+    return types.FunctionType(+        _cell_set_template_code,+        {},+        '_cell_set_inner',+        (),+        (cell,),+    )(value)+++#relevant opcodes+STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']+DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']+LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']+GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)+HAVE_ARGUMENT = dis.HAVE_ARGUMENT+EXTENDED_ARG = dis.EXTENDED_ARG+++def islambda(func):+    return getattr(func,'__name__') == '<lambda>'+++_BUILTIN_TYPE_NAMES = {}+for k, v in types.__dict__.items():+    if type(v) is type:+        _BUILTIN_TYPE_NAMES[v] = k+++def _builtin_type(name):+    return getattr(types, name)+++def _make__new__factory(type_):+    def _factory():+        return type_.__new__+    return _factory+++# NOTE: These need to be module globals so that they're pickleable as globals.+_get_dict_new = _make__new__factory(dict)+_get_frozenset_new = _make__new__factory(frozenset)+_get_list_new = _make__new__factory(list)+_get_set_new = _make__new__factory(set)+_get_tuple_new = _make__new__factory(tuple)+_get_object_new = _make__new__factory(object)++# Pre-defined set of builtin_function_or_method instances that can be+# serialized.+_BUILTIN_TYPE_CONSTRUCTORS = {+    dict.__new__: _get_dict_new,+    frozenset.__new__: _get_frozenset_new,+    set.__new__: _get_set_new,+    list.__new__: _get_list_new,+    tuple.__new__: _get_tuple_new,+    object.__new__: _get_object_new,+}+++if sys.version_info < (3, 4):+    def _walk_global_ops(code):+        """+        Yield (opcode, argument number) tuples for all+        global-referencing instructions in *code*.+        """+        code = getattr(code, 'co_code', b'')+        if not PY3:+            code = map(ord, code)++        n = len(code)+        i = 0+        extended_arg = 0+        while i < n:+            op = code[i]+            i += 1+            if op >= HAVE_ARGUMENT:+                oparg = code[i] + code[i + 1] * 256 + extended_arg+                extended_arg = 0+                i += 2+                if op == EXTENDED_ARG:+                    extended_arg = oparg * 65536+                if op in GLOBAL_OPS:+                    yield op, oparg++else:+    def _walk_global_ops(code):+        """+        Yield (opcode, argument number) tuples for all+        global-referencing instructions in *code*.+        """+        for instr in dis.get_instructions(code):+            op = instr.opcode+            if op in GLOBAL_OPS:+                yield op, instr.arg+++class CloudPickler(Pickler):++    dispatch = Pickler.dispatch.copy()++    def __init__(self, file, protocol=None):+        if protocol is None:+            protocol = DEFAULT_PROTOCOL+        Pickler.__init__(self, file, protocol=protocol)+        # set of modules to unpickle+        self.modules = set()+        # map ids to dictionary. used to ensure that functions can share global env+        self.globals_ref = {}++    def dump(self, obj):+        self.inject_addons()+        try:+            return Pickler.dump(self, obj)+        except RuntimeError as e:+            if 'recursion' in e.args[0]:+                msg = """Could not pickle object as excessively deep recursion required."""+                raise pickle.PicklingError(msg)++    def save_memoryview(self, obj):+        self.save(obj.tobytes())+    dispatch[memoryview] = save_memoryview++    if not PY3:+        def save_buffer(self, obj):+            self.save(str(obj))+        dispatch[buffer] = save_buffer++    def save_unsupported(self, obj):+        raise pickle.PicklingError("Cannot pickle objects of type %s" % type(obj))+    dispatch[types.GeneratorType] = save_unsupported++    # itertools objects do not pickle!+    for v in itertools.__dict__.values():+        if type(v) is type:+            dispatch[v] = save_unsupported++    def save_module(self, obj):+        """+        Save a module as an import+        """+        mod_name = obj.__name__+        # If module is successfully found then it is not a dynamically created module+        if hasattr(obj, '__file__'):+            is_dynamic = False+        else:+            try:+                _find_module(mod_name)+                is_dynamic = False+            except ImportError:+                is_dynamic = True++        self.modules.add(obj)+        if is_dynamic:+            self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)), obj=obj)+        else:+            self.save_reduce(subimport, (obj.__name__,), obj=obj)+    dispatch[types.ModuleType] = save_module++    def save_codeobject(self, obj):+        """+        Save a code object+        """+        if PY3:+            args = (+                obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,+                obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames,+                obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,+                obj.co_cellvars+            )+        else:+            args = (+                obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code,+                obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name,+                obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars+            )+        self.save_reduce(types.CodeType, args, obj=obj)+    dispatch[types.CodeType] = save_codeobject++    def save_function(self, obj, name=None):+        """ Registered with the dispatch to handle all function types.++        Determines what kind of function obj is (e.g. lambda, defined at+        interactive prompt, etc) and handles the pickling appropriately.+        """+        if obj in _BUILTIN_TYPE_CONSTRUCTORS:+            # We keep a special-cased cache of built-in type constructors at+            # global scope, because these functions are structured very+            # differently in different python versions and implementations (for+            # example, they're instances of types.BuiltinFunctionType in+            # CPython, but they're ordinary types.FunctionType instances in+            # PyPy).+            #+            # If the function we've received is in that cache, we just+            # serialize it as a lookup into the cache.+            return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj)++        write = self.write++        if name is None:+            name = obj.__name__+        try:+            # whichmodule() could fail, see+            # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling+            modname = pickle.whichmodule(obj, name)+        except Exception:+            modname = None+        # print('which gives %s %s %s' % (modname, obj, name))+        try:+            themodule = sys.modules[modname]+        except KeyError:+            # eval'd items such as namedtuple give invalid items for their function __module__+            modname = '__main__'++        if modname == '__main__':+            themodule = None++        try:+            lookedup_by_name = getattr(themodule, name, None)+        except Exception:+            lookedup_by_name = None++        if themodule:+            self.modules.add(themodule)+            if lookedup_by_name is obj:+                return self.save_global(obj, name)++        # a builtin_function_or_method which comes in as an attribute of some+        # object (e.g., itertools.chain.from_iterable) will end+        # up with modname "__main__" and so end up here. But these functions+        # have no __code__ attribute in CPython, so the handling for+        # user-defined functions below will fail.+        # So we pickle them here using save_reduce; have to do it differently+        # for different python versions.+        if not hasattr(obj, '__code__'):+            if PY3:+                rv = obj.__reduce_ex__(self.proto)+            else:+                if hasattr(obj, '__self__'):+                    rv = (getattr, (obj.__self__, name))+                else:+                    raise pickle.PicklingError("Can't pickle %r" % obj)+            return self.save_reduce(obj=obj, *rv)++        # if func is lambda, def'ed at prompt, is in main, or is nested, then+        # we'll pickle the actual function object rather than simply saving a+        # reference (as is done in default pickler), via save_function_tuple.+        if (islambda(obj)+                or getattr(obj.__code__, 'co_filename', None) == '<stdin>'+                or themodule is None):+            self.save_function_tuple(obj)+            return+        else:+            # func is nested+            if lookedup_by_name is None or lookedup_by_name is not obj:+                self.save_function_tuple(obj)+                return++        if obj.__dict__:+            # essentially save_reduce, but workaround needed to avoid recursion+            self.save(_restore_attr)+            write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')+            self.memoize(obj)+            self.save(obj.__dict__)+            write(pickle.TUPLE + pickle.REDUCE)+        else:+            write(pickle.GLOBAL + modname + '\n' + name + '\n')+            self.memoize(obj)+    dispatch[types.FunctionType] = save_function++    def _save_subimports(self, code, top_level_dependencies):+        """+        Ensure de-pickler imports any package child-modules that+        are needed by the function+        """+        # check if any known dependency is an imported package+        for x in top_level_dependencies:+            if isinstance(x, types.ModuleType) and hasattr(x, '__package__') and x.__package__:+                # check if the package has any currently loaded sub-imports+                prefix = x.__name__ + '.'+                for name, module in sys.modules.items():+                    # Older versions of pytest will add a "None" module to sys.modules.+                    if name is not None and name.startswith(prefix):+                        # check whether the function can address the sub-module+                        tokens = set(name[len(prefix):].split('.'))+                        if not tokens - set(code.co_names):+                            # ensure unpickler executes this import+                            self.save(module)+                            # then discards the reference to it+                            self.write(pickle.POP)++    def save_dynamic_class(self, obj):+        """+        Save a class that can't be stored as module global.++        This method is used to serialize classes that are defined inside+        functions, or that otherwise can't be serialized as attribute lookups+        from global modules.+        """+        clsdict = dict(obj.__dict__)  # copy dict proxy to a dict+        clsdict.pop('__weakref__', None)++        # On PyPy, __doc__ is a readonly attribute, so we need to include it in+        # the initial skeleton class.  This is safe because we know that the+        # doc can't participate in a cycle with the original class.+        type_kwargs = {'__doc__': clsdict.pop('__doc__', None)}++        # If type overrides __dict__ as a property, include it in the type kwargs.+        # In Python 2, we can't set this attribute after construction.+        __dict__ = clsdict.pop('__dict__', None)+        if isinstance(__dict__, property):+            type_kwargs['__dict__'] = __dict__++        save = self.save+        write = self.write++        # We write pickle instructions explicitly here to handle the+        # possibility that the type object participates in a cycle with its own+        # __dict__. We first write an empty "skeleton" version of the class and+        # memoize it before writing the class' __dict__ itself. We then write+        # instructions to "rehydrate" the skeleton class by restoring the+        # attributes from the __dict__.+        #+        # A type can appear in a cycle with its __dict__ if an instance of the+        # type appears in the type's __dict__ (which happens for the stdlib+        # Enum class), or if the type defines methods that close over the name+        # of the type, (which is common for Python 2-style super() calls).++        # Push the rehydration function.+        save(_rehydrate_skeleton_class)++        # Mark the start of the args tuple for the rehydration function.+        write(pickle.MARK)++        # Create and memoize an skeleton class with obj's name and bases.+        tp = type(obj)+        self.save_reduce(tp, (obj.__name__, obj.__bases__, type_kwargs), obj=obj)++        # Now save the rest of obj's __dict__. Any references to obj+        # encountered while saving will point to the skeleton class.+        save(clsdict)++        # Write a tuple of (skeleton_class, clsdict).+        write(pickle.TUPLE)++        # Call _rehydrate_skeleton_class(skeleton_class, clsdict)+        write(pickle.REDUCE)++    def save_function_tuple(self, func):+        """  Pickles an actual func object.++        A func comprises: code, globals, defaults, closure, and dict.  We+        extract and save these, injecting reducing functions at certain points+        to recreate the func object.  Keep in mind that some of these pieces+        can contain a ref to the func itself.  Thus, a naive save on these+        pieces could trigger an infinite loop of save's.  To get around that,+        we first create a skeleton func object using just the code (this is+        safe, since this won't contain a ref to the func), and memoize it as+        soon as it's created.  The other stuff can then be filled in later.+        """+        if is_tornado_coroutine(func):+            self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,),+                             obj=func)+            return++        save = self.save+        write = self.write++        code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func)++        save(_fill_function)  # skeleton function updater+        write(pickle.MARK)    # beginning of tuple that _fill_function expects++        self._save_subimports(+            code,+            itertools.chain(f_globals.values(), closure_values or ()),+        )++        # create a skeleton function object and memoize it+        save(_make_skel_func)+        save((+            code,+            len(closure_values) if closure_values is not None else -1,+            base_globals,+        ))+        write(pickle.REDUCE)+        self.memoize(func)++        # save the rest of the func data needed by _fill_function+        state = {+            'globals': f_globals,+            'defaults': defaults,+            'dict': dct,+            'module': func.__module__,+            'closure_values': closure_values,+        }+        if hasattr(func, '__qualname__'):+            state['qualname'] = func.__qualname__+        save(state)+        write(pickle.TUPLE)+        write(pickle.REDUCE)  # applies _fill_function on the tuple++    _extract_code_globals_cache = (+        weakref.WeakKeyDictionary()+        if not hasattr(sys, "pypy_version_info")+        else {})++    @classmethod+    def extract_code_globals(cls, co):+        """+        Find all globals names read or written to by codeblock co+        """+        out_names = cls._extract_code_globals_cache.get(co)+        if out_names is None:+            try:+                names = co.co_names+            except AttributeError:+                # PyPy "builtin-code" object+                out_names = set()+            else:+                out_names = set(names[oparg]+                                for op, oparg in _walk_global_ops(co))++                # see if nested function have any global refs+                if co.co_consts:+                    for const in co.co_consts:+                        if type(const) is types.CodeType:+                            out_names |= cls.extract_code_globals(const)++            cls._extract_code_globals_cache[co] = out_names++        return out_names++    def extract_func_data(self, func):+        """+        Turn the function into a tuple of data necessary to recreate it:+            code, globals, defaults, closure_values, dict+        """+        code = func.__code__++        # extract all global ref's+        func_global_refs = self.extract_code_globals(code)++        # process all variables referenced by global environment+        f_globals = {}+        for var in func_global_refs:+            if var in func.__globals__:+                f_globals[var] = func.__globals__[var]++        # defaults requires no processing+        defaults = func.__defaults__++        # process closure+        closure = (+            list(map(_get_cell_contents, func.__closure__))+            if func.__closure__ is not None+            else None+        )++        # save the dict+        dct = func.__dict__++        base_globals = self.globals_ref.get(id(func.__globals__), {})+        self.globals_ref[id(func.__globals__)] = base_globals++        return (code, f_globals, defaults, closure, dct, base_globals)++    def save_builtin_function(self, obj):+        if obj.__module__ == "__builtin__":+            return self.save_global(obj)+        return self.save_function(obj)+    dispatch[types.BuiltinFunctionType] = save_builtin_function++    def save_global(self, obj, name=None, pack=struct.pack):+        """+        Save a "global".++        The name of this method is somewhat misleading: all types get+        dispatched here.+        """+        if obj.__module__ == "__main__":+            return self.save_dynamic_class(obj)++        try:+            return Pickler.save_global(self, obj, name=name)+        except Exception:+            if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":+                if obj in _BUILTIN_TYPE_NAMES:+                    return self.save_reduce(+                        _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)++            typ = type(obj)+            if typ is not obj and isinstance(obj, (type, types.ClassType)):+                return self.save_dynamic_class(obj)++            raise++    dispatch[type] = save_global+    dispatch[types.ClassType] = save_global++    def save_instancemethod(self, obj):+        # Memoization rarely is ever useful due to python bounding+        if obj.__self__ is None:+            self.save_reduce(getattr, (obj.im_class, obj.__name__))+        else:+            if PY3:+                self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj)+            else:+                self.save_reduce(types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__),+                         obj=obj)+    dispatch[types.MethodType] = save_instancemethod++    def save_inst(self, obj):+        """Inner logic to save instance. Based off pickle.save_inst"""+        cls = obj.__class__++        # Try the dispatch table (pickle module doesn't do it)+        f = self.dispatch.get(cls)+        if f:+            f(self, obj)  # Call unbound method with explicit self+            return++        memo = self.memo+        write = self.write+        save = self.save++        if hasattr(obj, '__getinitargs__'):+            args = obj.__getinitargs__()+            len(args)  # XXX Assert it's a sequence+            pickle._keep_alive(args, memo)+        else:+            args = ()++        write(pickle.MARK)++        if self.bin:+            save(cls)+            for arg in args:+                save(arg)+            write(pickle.OBJ)+        else:+            for arg in args:+                save(arg)+            write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n')++        self.memoize(obj)++        try:+            getstate = obj.__getstate__+        except AttributeError:+            stuff = obj.__dict__+        else:+            stuff = getstate()+            pickle._keep_alive(stuff, memo)+        save(stuff)+        write(pickle.BUILD)++    if not PY3:+        dispatch[types.InstanceType] = save_inst++    def save_property(self, obj):+        # properties not correctly saved in python+        self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj)+    dispatch[property] = save_property++    def save_classmethod(self, obj):+        orig_func = obj.__func__+        self.save_reduce(type(obj), (orig_func,), obj=obj)+    dispatch[classmethod] = save_classmethod+    dispatch[staticmethod] = save_classmethod++    def save_itemgetter(self, obj):+        """itemgetter serializer (needed for namedtuple support)"""+        class Dummy:+            def __getitem__(self, item):+                return item+        items = obj(Dummy())+        if not isinstance(items, tuple):+            items = (items, )+        return self.save_reduce(operator.itemgetter, items)++    if type(operator.itemgetter) is type:+        dispatch[operator.itemgetter] = save_itemgetter++    def save_attrgetter(self, obj):+        """attrgetter serializer"""+        class Dummy(object):+            def __init__(self, attrs, index=None):+                self.attrs = attrs+                self.index = index+            def __getattribute__(self, item):+                attrs = object.__getattribute__(self, "attrs")+                index = object.__getattribute__(self, "index")+                if index is None:+                    index = len(attrs)+                    attrs.append(item)+                else:+                    attrs[index] = ".".join([attrs[index], item])+                return type(self)(attrs, index)+        attrs = []+        obj(Dummy(attrs))+        return self.save_reduce(operator.attrgetter, tuple(attrs))++    if type(operator.attrgetter) is type:+        dispatch[operator.attrgetter] = save_attrgetter++    def save_file(self, obj):+        """Save a file"""+        try:+            import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute+        except ImportError:+            import io as pystringIO++        if not hasattr(obj, 'name') or  not hasattr(obj, 'mode'):+            raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")+        if obj is sys.stdout:+            return self.save_reduce(getattr, (sys,'stdout'), obj=obj)+        if obj is sys.stderr:+            return self.save_reduce(getattr, (sys,'stderr'), obj=obj)+        if obj is sys.stdin:+            raise pickle.PicklingError("Cannot pickle standard input")+        if obj.closed:+            raise pickle.PicklingError("Cannot pickle closed files")+        if hasattr(obj, 'isatty') and obj.isatty():+            raise pickle.PicklingError("Cannot pickle files that map to tty objects")+        if 'r' not in obj.mode and '+' not in obj.mode:+            raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode)++        name = obj.name++        retval = pystringIO.StringIO()++        try:+            # Read the whole file+            curloc = obj.tell()+            obj.seek(0)+            contents = obj.read()+            obj.seek(curloc)+        except IOError:+            raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)+        retval.write(contents)+        retval.seek(curloc)++        retval.name = name+        self.save(retval)+        self.memoize(obj)++    def save_ellipsis(self, obj):+        self.save_reduce(_gen_ellipsis, ())++    def save_not_implemented(self, obj):+        self.save_reduce(_gen_not_implemented, ())++    if PY3:+        dispatch[io.TextIOWrapper] = save_file+    else:+        dispatch[file] = save_file++    dispatch[type(Ellipsis)] = save_ellipsis+    dispatch[type(NotImplemented)] = save_not_implemented++    def save_weakset(self, obj):+        self.save_reduce(weakref.WeakSet, (list(obj),))++    dispatch[weakref.WeakSet] = save_weakset++    def save_logger(self, obj):+        self.save_reduce(logging.getLogger, (obj.name,), obj=obj)++    dispatch[logging.Logger] = save_logger++    """Special functions for Add-on libraries"""+    def inject_addons(self):+        """Plug in system. Register additional pickling functions if modules already loaded"""+        pass+++# Tornado support++def is_tornado_coroutine(func):+    """+    Return whether *func* is a Tornado coroutine function.+    Running coroutines are not supported.+    """+    if 'tornado.gen' not in sys.modules:+        return False+    gen = sys.modules['tornado.gen']+    if not hasattr(gen, "is_coroutine_function"):+        # Tornado version is too old+        return False+    return gen.is_coroutine_function(func)+++def _rebuild_tornado_coroutine(func):+    from tornado import gen+    return gen.coroutine(func)+++# Shorthands for legacy support++def dump(obj, file, protocol=None):+    """Serialize obj as bytes streamed into file++    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to+    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed+    between processes running the same Python version.++    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure+    compatibility with older versions of Python.+    """+    CloudPickler(file, protocol=protocol).dump(obj)+++def dumps(obj, protocol=None):+    """Serialize obj as a string of bytes allocated in memory++    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to+    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed+    between processes running the same Python version.++    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure+    compatibility with older versions of Python.+    """+    file = StringIO()+    try:+        cp = CloudPickler(file, protocol=protocol)+        cp.dump(obj)+        return file.getvalue()+    finally:+        file.close()+++# including pickles unloading functions in this namespace+load = pickle.load+loads = pickle.loads+++# hack for __import__ not working as desired+def subimport(name):+    __import__(name)+    return sys.modules[name]+++def dynamic_subimport(name, vars):+    mod = imp.new_module(name)+    mod.__dict__.update(vars)+    sys.modules[name] = mod+    return mod+++# restores function attributes+def _restore_attr(obj, attr):+    for key, val in attr.items():+        setattr(obj, key, val)+    return obj+++def _get_module_builtins():+    return pickle.__builtins__+++def print_exec(stream):+    ei = sys.exc_info()+    traceback.print_exception(ei[0], ei[1], ei[2], None, stream)+++def _modules_to_main(modList):+    """Force every module in modList to be placed into main"""+    if not modList:+        return++    main = sys.modules['__main__']+    for modname in modList:+        if type(modname) is str:+            try:+                mod = __import__(modname)+            except Exception as e:+                sys.stderr.write('warning: could not import %s\n.  '+                                 'Your function may unexpectedly error due to this import failing;'+                                 'A version mismatch is likely.  Specific error was:\n' % modname)+                print_exec(sys.stderr)+            else:+                setattr(main, mod.__name__, mod)+++#object generators:+def _genpartial(func, args, kwds):+    if not args:+        args = ()+    if not kwds:+        kwds = {}+    return partial(func, *args, **kwds)++def _gen_ellipsis():+    return Ellipsis++def _gen_not_implemented():+    return NotImplemented+++def _get_cell_contents(cell):+    try:+        return cell.cell_contents+    except ValueError:+        # sentinel used by ``_fill_function`` which will leave the cell empty+        return _empty_cell_value+++def instance(cls):+    """Create a new instance of a class.++    Parameters+    ----------+    cls : type+        The class to create an instance of.++    Returns+    -------+    instance : cls+        A new instance of ``cls``.+    """+    return cls()+++@instance+class _empty_cell_value(object):+    """sentinel for empty closures+    """+    @classmethod+    def __reduce__(cls):+        return cls.__name__+++def _fill_function(*args):+    """Fills in the rest of function data into the skeleton function object++    The skeleton itself is create by _make_skel_func().+    """+    if len(args) == 2:+        func = args[0]+        state = args[1]+    elif len(args) == 5:+        # Backwards compat for cloudpickle v0.4.0, after which the `module`+        # argument was introduced+        func = args[0]+        keys = ['globals', 'defaults', 'dict', 'closure_values']+        state = dict(zip(keys, args[1:]))+    elif len(args) == 6:+        # Backwards compat for cloudpickle v0.4.1, after which the function+        # state was passed as a dict to the _fill_function it-self.+        func = args[0]+        keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']+        state = dict(zip(keys, args[1:]))+    else:+        raise ValueError('Unexpected _fill_value arguments: %r' % (args,))++    func.__globals__.update(state['globals'])+    func.__defaults__ = state['defaults']+    func.__dict__ = state['dict']+    if 'module' in state:+        func.__module__ = state['module']+    if 'qualname' in state:+        func.__qualname__ = state['qualname']++    cells = func.__closure__+    if cells is not None:+        for cell, value in zip(cells, state['closure_values']):+            if value is not _empty_cell_value:+                cell_set(cell, value)++    return func+++def _make_empty_cell():+    if False:+        # trick the compiler into creating an empty cell in our lambda+        cell = None+        raise AssertionError('this route should not be executed')++    return (lambda: cell).__closure__[0]+++def _make_skel_func(code, cell_count, base_globals=None):+    """ Creates a skeleton function object that contains just the provided+        code and the correct number of cells in func_closure.  All other+        func attributes (e.g. func_globals) are empty.+    """+    if base_globals is None:+        base_globals = {}+    base_globals['__builtins__'] = __builtins__++    closure = (+        tuple(_make_empty_cell() for _ in range(cell_count))+        if cell_count >= 0 else+        None+    )+    return types.FunctionType(code, base_globals, None, None, closure)+++def _rehydrate_skeleton_class(skeleton_class, class_dict):+    """Put attributes from `class_dict` back on `skeleton_class`.++    See CloudPickler.save_dynamic_class for more info.+    """+    for attrname, attr in class_dict.items():+        setattr(skeleton_class, attrname, attr)+    return skeleton_class+++def _find_module(mod_name):+    """+    Iterate over each part instead of calling imp.find_module directly.+    This function is able to find submodules (e.g. sickit.tree)+    """+    path = None+    for part in mod_name.split('.'):+        if path is not None:+            path = [path]+        file, path, description = imp.find_module(part, path)+        if file is not None:+            file.close()+    return path, description++"""Constructors for 3rd party libraries+Note: These can never be renamed due to client compatibility issues"""++def _getobject(modname, attribute):+    mod = __import__(modname, fromlist=[attribute])+    return mod.__dict__[attribute]+++""" Use copy_reg to extend global pickle definitions """++if sys.version_info < (3, 4):+    method_descriptor = type(str.upper)++    def _reduce_method_descriptor(obj):+        return (getattr, (obj.__objclass__, obj.__name__))++    try:+        import copy_reg as copyreg+    except ImportError:+        import copyreg+    copyreg.pickle(method_descriptor, _reduce_method_descriptor)
+ test/files/joblib2.py view
@@ -0,0 +1,1084 @@+###############################################################################+# Re-implementation of the ProcessPoolExecutor more robust to faults+#+# author: Thomas Moreau and Olivier Grisel+#+# adapted from concurrent/futures/process_pool_executor.py (17/02/2017)+#  * Backport for python2.7/3.3,+#  * Add an extra management thread to detect queue_management_thread failures,+#  * Improve the shutdown process to avoid deadlocks,+#  * Add timeout for workers,+#  * More robust pickling process.+#+# Copyright 2009 Brian Quinlan. All Rights Reserved.+# Licensed to PSF under a Contributor Agreement.++"""Implements ProcessPoolExecutor.++The follow diagram and text describe the data-flow through the system:++|======================= In-process =====================|== Out-of-process ==|+++----------+     +----------+       +--------+     +-----------+    +---------++|          |  => | Work Ids |       |        |     | Call Q    |    | Process |+|          |     +----------+       |        |     +-----------+    |  Pool   |+|          |     | ...      |       |        |     | ...       |    +---------++|          |     | 6        |    => |        |  => | 5, call() | => |         |+|          |     | 7        |       |        |     | ...       |    |         |+| Process  |     | ...      |       | Local  |     +-----------+    | Process |+|  Pool    |     +----------+       | Worker |                      |  #1..n  |+| Executor |                        | Thread |                      |         |+|          |     +----------- +     |        |     +-----------+    |         |+|          | <=> | Work Items | <=> |        | <=  | Result Q  | <= |         |+|          |     +------------+     |        |     +-----------+    |         |+|          |     | 6: call()  |     |        |     | ...       |    |         |+|          |     |    future  |     +--------+     | 4, result |    |         |+|          |     | ...        |                    | 3, except |    |         |++----------+     +------------+                    +-----------+    +---------+++Executor.submit() called:+- creates a uniquely numbered _WorkItem and adds it to the "Work Items" dict+- adds the id of the _WorkItem to the "Work Ids" queue++Local worker thread:+- reads work ids from the "Work Ids" queue and looks up the corresponding+  WorkItem from the "Work Items" dict: if the work item has been cancelled then+  it is simply removed from the dict, otherwise it is repackaged as a+  _CallItem and put in the "Call Q". New _CallItems are put in the "Call Q"+  until "Call Q" is full. NOTE: the size of the "Call Q" is kept small because+  calls placed in the "Call Q" can no longer be cancelled with Future.cancel().+- reads _ResultItems from "Result Q", updates the future stored in the+  "Work Items" dict and deletes the dict entry++Process #1..n:+- reads _CallItems from "Call Q", executes the calls, and puts the resulting+  _ResultItems in "Result Q"+"""+++__author__ = 'Thomas Moreau (thomas.moreau.2010@gmail.com)'+++import os+import sys+import types+import weakref+import warnings+import itertools+import traceback+import threading+import multiprocessing as mp+from functools import partial+from pickle import PicklingError+from time import time+import gc++from . import _base+from .backend import get_context+from .backend.compat import queue+from .backend.compat import wait+from .backend.context import cpu_count+from .backend.queues import Queue, SimpleQueue, Full+from .backend.utils import recursive_terminate++try:+    from concurrent.futures.process import BrokenProcessPool as _BPPException+except ImportError:+    _BPPException = RuntimeError+++# Compatibility for python2.7+if sys.version_info[0] == 2:+    ProcessLookupError = OSError+++# Workers are created as daemon threads and processes. This is done to allow+# the interpreter to exit when there are still idle processes in a+# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However,+# allowing workers to die with the interpreter has two undesirable properties:+#   - The workers would still be running during interpreter shutdown,+#     meaning that they would fail in unpredictable ways.+#   - The workers could be killed while evaluating a work item, which could+#     be bad if the callable being evaluated has external side-effects e.g.+#     writing to a file.+#+# To work around this problem, an exit handler is installed which tells the+# workers to exit when their work queues are empty and then waits until the+# threads/processes finish.++_threads_wakeups = weakref.WeakKeyDictionary()+_global_shutdown = False++# Mechanism to prevent infinite process spawning. When a worker of a+# ProcessPoolExecutor nested in MAX_DEPTH Executor tries to create a new+# Executor, a LokyRecursionError is raised+MAX_DEPTH = int(os.environ.get("LOKY_MAX_DEPTH", 10))+_CURRENT_DEPTH = 0++# Minimum time interval between two consecutive memory usage checks.+_MEMORY_CHECK_DELAY = 1.++# Number of bytes of memory usage allowed over the reference process size.+_MAX_MEMORY_LEAK_SIZE = int(1e8)++try:+    from psutil import Process++    def _get_memory_usage(pid, force_gc=False):+        if force_gc:+            gc.collect()++        return Process(pid).memory_info().rss++except ImportError:+    _get_memory_usage = None+++class _ThreadWakeup:+    def __init__(self):+        self._reader, self._writer = mp.Pipe(duplex=False)++    def close(self):+        self._writer.close()+        self._reader.close()++    def wakeup(self):+        if sys.platform == "win32" and sys.version_info[:2] < (3, 4):+            # Compat for python2.7 on windows, where poll return false for+            # b"" messages. Use the slightly larger message b"0".+            self._writer.send_bytes(b"0")+        else:+            self._writer.send_bytes(b"")++    def clear(self):+        while self._reader.poll():+            self._reader.recv_bytes()+++class _ExecutorFlags(object):+    """necessary references to maintain executor states without preventing gc++    It permits to keep the information needed by queue_management_thread+    and crash_detection_thread to maintain the pool without preventing the+    garbage collection of unreferenced executors.+    """+    def __init__(self):++        self.shutdown = False+        self.broken = None+        self.kill_workers = False+        self.shutdown_lock = threading.Lock()++    def flag_as_shutting_down(self, kill_workers=False):+        with self.shutdown_lock:+            self.shutdown = True+            self.kill_workers = kill_workers++    def flag_as_broken(self, broken):+        with self.shutdown_lock:+            self.shutdown = True+            self.broken = broken+++def _python_exit():+    global _global_shutdown+    _global_shutdown = True+    items = list(_threads_wakeups.items())+    mp.util.debug("Interpreter shutting down. Waking up queue_manager_threads "+                  "{}".format(items))+    for thread, thread_wakeup in items:+        if thread.is_alive():+            thread_wakeup.wakeup()+    for thread, _ in items:+        thread.join()+++# Module variable to register the at_exit call+process_pool_executor_at_exit = None++# Controls how many more calls than processes will be queued in the call queue.+# A smaller number will mean that processes spend more time idle waiting for+# work while a larger number will make Future.cancel() succeed less frequently+# (Futures in the call queue cannot be cancelled).+EXTRA_QUEUED_CALLS = 1+++class _RemoteTraceback(Exception):+    """Embed stringification of remote traceback in local traceback+    """+    def __init__(self, tb=None):+        self.tb = tb++    def __str__(self):+        return self.tb+++class _ExceptionWithTraceback(BaseException):++    def __init__(self, exc, tb=None):+        if tb is None:+            _, _, tb = sys.exc_info()+        tb = traceback.format_exception(type(exc), exc, tb)+        tb = ''.join(tb)+        self.exc = exc+        self.tb = '\n"""\n%s"""' % tb++    def __reduce__(self):+        return _rebuild_exc, (self.exc, self.tb)+++def _rebuild_exc(exc, tb):+    exc.__cause__ = _RemoteTraceback(tb)+    return exc+++class _WorkItem(object):++    __slots__ = ["future", "fn", "args", "kwargs"]++    def __init__(self, future, fn, args, kwargs):+        self.future = future+        self.fn = fn+        self.args = args+        self.kwargs = kwargs+++class _ResultItem(object):++    def __init__(self, work_id, exception=None, result=None):+        self.work_id = work_id+        self.exception = exception+        self.result = result+++class _CallItem(object):++    def __init__(self, work_id, fn, args, kwargs):+        self.work_id = work_id+        self.fn = fn+        self.args = args+        self.kwargs = kwargs++    def __repr__(self):+        return "CallItem({}, {}, {}, {})".format(+            self.work_id, self.fn, self.args, self.kwargs)++    try:+        # If cloudpickle is present on the system, use it to pickle the+        # function. This permits to use interactive terminal for loky calls.+        # TODO: Add option to deactivate, as it increases pickling time.+        from .backend import LOKY_PICKLER+        assert LOKY_PICKLER is None or LOKY_PICKLER == ""++        import cloudpickle  # noqa: F401++        def __getstate__(self):+            from cloudpickle import dumps+            if isinstance(self.fn, (types.FunctionType,+                                    types.LambdaType,+                                    partial)):+                cp = True+                fn = dumps(self.fn)+            else:+                cp = False+                fn = self.fn+            return (self.work_id, self.args, self.kwargs, fn, cp)++        def __setstate__(self, state):+            self.work_id, self.args, self.kwargs, self.fn, cp = state+            if cp:+                from cloudpickle import loads+                self.fn = loads(self.fn)++    except (ImportError, AssertionError) as e:+        pass+++class _SafeQueue(Queue):+    """Safe Queue set exception to the future object linked to a job"""+    def __init__(self, max_size=0, ctx=None, pending_work_items=None,+                 running_work_items=None, thread_wakeup=None, reducers=None):+        self.thread_wakeup = thread_wakeup+        self.pending_work_items = pending_work_items+        self.running_work_items = running_work_items+        super(_SafeQueue, self).__init__(max_size, reducers=reducers, ctx=ctx)++    def _on_queue_feeder_error(self, e, obj):+        if isinstance(obj, _CallItem):+            # fromat traceback only on python3+            pickling_error = PicklingError(+                "Could not pickle the task to send it to the workers.")+            tb = traceback.format_exception(+                type(e), e, getattr(e, "__traceback__", None))+            pickling_error.__cause__ = _RemoteTraceback(+                '\n"""\n{}"""'.format(''.join(tb)))+            work_item = self.pending_work_items.pop(obj.work_id, None)+            self.running_work_items.remove(obj.work_id)+            # work_item can be None if another process terminated. In this+            # case, the queue_manager_thread fails all work_items with+            # BrokenProcessPool+            if work_item is not None:+                work_item.future.set_exception(pickling_error)+                del work_item+            self.thread_wakeup.wakeup()+        else:+            super()._on_queue_feeder_error(e, obj)+++def _get_chunks(chunksize, *iterables):+    """ Iterates over zip()ed iterables in chunks. """+    if sys.version_info < (3, 3):+        it = itertools.izip(*iterables)+    else:+        it = zip(*iterables)+    while True:+        chunk = tuple(itertools.islice(it, chunksize))+        if not chunk:+            return+        yield chunk+++def _process_chunk(fn, chunk):+    """ Processes a chunk of an iterable passed to map.++    Runs the function passed to map() on a chunk of the+    iterable passed to map.++    This function is run in a separate process.++    """+    return [fn(*args) for args in chunk]+++def _sendback_result(result_queue, work_id, result=None, exception=None):+    """Safely send back the given result or exception"""+    try:+        result_queue.put(_ResultItem(work_id, result=result,+                                     exception=exception))+    except BaseException as e:+        exc = _ExceptionWithTraceback(e, getattr(e, "__traceback__", None))+        result_queue.put(_ResultItem(work_id, exception=exc))+++def _process_worker(call_queue, result_queue, initializer, initargs,+                    processes_management_lock, timeout, worker_exit_lock,+                    current_depth):+    """Evaluates calls from call_queue and places the results in result_queue.++    This worker is run in a separate process.++    Args:+        call_queue: A ctx.Queue of _CallItems that will be read and+            evaluated by the worker.+        result_queue: A ctx.Queue of _ResultItems that will written+            to by the worker.+        initializer: A callable initializer, or None+        initargs: A tuple of args for the initializer+        process_management_lock: A ctx.Lock avoiding worker timeout while some+            workers are being spawned.+        timeout: maximum time to wait for a new item in the call_queue. If that+            time is expired, the worker will shutdown.+        worker_exit_lock: Lock to avoid flagging the executor as broken on+            workers timeout.+        current_depth: Nested parallelism level, to avoid infinite spawning.+    """+    if initializer is not None:+        try:+            initializer(*initargs)+        except BaseException:+            _base.LOGGER.critical('Exception in initializer:', exc_info=True)+            # The parent will notice that the process stopped and+            # mark the pool broken+            return++    # set the global _CURRENT_DEPTH mechanism to limit recursive call+    global _CURRENT_DEPTH+    _CURRENT_DEPTH = current_depth+    _REFERENCE_PROCESS_SIZE = None+    _LAST_MEMORY_CHECK = None+    pid = os.getpid()++    mp.util.debug('Worker started with timeout=%s' % timeout)+    while True:+        try:+            call_item = call_queue.get(block=True, timeout=timeout)+            if call_item is None:+                mp.util.info("Shutting down worker on sentinel")+        except queue.Empty:+            mp.util.info("Shutting down worker after timeout %0.3fs"+                         % timeout)+            if processes_management_lock.acquire(block=False):+                processes_management_lock.release()+                call_item = None+            else:+                mp.util.info("Could not acquire processes_management_lock")+                continue+        except BaseException as e:+            traceback.print_exc()+            sys.exit(1)+        if call_item is None:+            # Notify queue management thread about clean worker shutdown+            result_queue.put(pid)+            with worker_exit_lock:+                return+        try:+            r = call_item.fn(*call_item.args, **call_item.kwargs)+        except BaseException as e:+            exc = _ExceptionWithTraceback(e, getattr(e, "__traceback__", None))+            result_queue.put(_ResultItem(call_item.work_id, exception=exc))+        else:+            _sendback_result(result_queue, call_item.work_id, result=r)++        # Free the resource as soon as possible, to avoid holding onto+        # open files or shared memory that is not needed anymore+        del call_item++        if _get_memory_usage is not None:+            if _REFERENCE_PROCESS_SIZE is None:+                # Make reference measurement after the first call+                _REFERENCE_PROCESS_SIZE = _get_memory_usage(pid, force_gc=True)+                _LAST_MEMORY_CHECK = time()+                continue+            if time() - _LAST_MEMORY_CHECK > _MEMORY_CHECK_DELAY:+                mem_usage = _get_memory_usage(pid)+                _LAST_MEMORY_CHECK = time()+                if mem_usage - _REFERENCE_PROCESS_SIZE < _MAX_MEMORY_LEAK_SIZE:+                    # Memory usage stays within bounds: everything is fine.+                    continue++                # Check again memory usage; this time take the measurement+                # after a forced garbage collection to break any reference+                # cycles.+                mem_usage = _get_memory_usage(pid, force_gc=True)+                _LAST_MEMORY_CHECK = time()+                if mem_usage - _REFERENCE_PROCESS_SIZE < _MAX_MEMORY_LEAK_SIZE:+                    # The GC managed to free the memory: everything is fine.+                    continue++                # The process is leaking memory: let the master process+                # know that we need to start a new worker.+                mp.util.info("Memory leak detected: shutting down worker")+                result_queue.put(pid)+                with worker_exit_lock:+                    return+++def _add_call_item_to_queue(pending_work_items,+                            running_work_items,+                            work_ids,+                            call_queue):+    """Fills call_queue with _WorkItems from pending_work_items.++    This function never blocks.++    Args:+        pending_work_items: A dict mapping work ids to _WorkItems e.g.+            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}+        work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids+            are consumed and the corresponding _WorkItems from+            pending_work_items are transformed into _CallItems and put in+            call_queue.+        call_queue: A ctx.Queue that will be filled with _CallItems+            derived from _WorkItems.+    """+    while True:+        if call_queue.full():+            return+        try:+            work_id = work_ids.get(block=False)+        except queue.Empty:+            return+        else:+            work_item = pending_work_items[work_id]++            if work_item.future.set_running_or_notify_cancel():+                running_work_items += [work_id]+                call_queue.put(_CallItem(work_id,+                                         work_item.fn,+                                         work_item.args,+                                         work_item.kwargs),+                               block=True)+            else:+                del pending_work_items[work_id]+                continue+++def _queue_management_worker(executor_reference,+                             executor_flags,+                             processes,+                             pending_work_items,+                             running_work_items,+                             work_ids_queue,+                             call_queue,+                             result_queue,+                             thread_wakeup,+                             processes_management_lock):+    """Manages the communication between this process and the worker processes.++    This function is run in a local thread.++    Args:+        executor_reference: A weakref.ref to the ProcessPoolExecutor that owns+            this thread. Used to determine if the ProcessPoolExecutor has been+            garbage collected and that this function can exit.+        executor_flags: A ExecutorFlags holding internal states of the+            ProcessPoolExecutor. It permits to know if the executor is broken+            even the object has been gc.+        process: A list of the ctx.Process instances used as+            workers.+        pending_work_items: A dict mapping work ids to _WorkItems e.g.+            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}+        work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]).+        call_queue: A ctx.Queue that will be filled with _CallItems+            derived from _WorkItems for processing by the process workers.+        result_queue: A ctx.SimpleQueue of _ResultItems generated by the+            process workers.+        thread_wakeup: A _ThreadWakeup to allow waking up the+            queue_manager_thread from the main Thread and avoid deadlocks+            caused by permanently locked queues.+    """+    executor = None++    def is_shutting_down():+        # No more work items can be added if:+        #   - The interpreter is shutting down OR+        #   - The executor that own this worker is not broken AND+        #        * The executor that owns this worker has been collected OR+        #        * The executor that owns this worker has been shutdown.+        # If the executor is broken, it should be detected in the next loop.+        return (_global_shutdown or+                ((executor is None or executor_flags.shutdown)+                 and not executor_flags.broken))++    def shutdown_all_workers():+        mp.util.debug("queue management thread shutting down")+        executor_flags.flag_as_shutting_down()+        # Create a list to avoid RuntimeError due to concurrent modification of+        # processes. nb_children_alive is thus an upper bound. Also release the+        # processes' _worker_exit_lock to accelerate the shutdown procedure, as+        # there is no need for hand-shake here.+        with processes_management_lock:+            n_children_alive = 0+            for p in list(processes.values()):+                p._worker_exit_lock.release()+                n_children_alive += 1+        n_children_to_stop = n_children_alive+        n_sentinels_sent = 0+        # Send the right number of sentinels, to make sure all children are+        # properly terminated.+        while n_sentinels_sent < n_children_to_stop and n_children_alive > 0:+            for i in range(n_children_to_stop - n_sentinels_sent):+                try:+                    call_queue.put_nowait(None)+                    n_sentinels_sent += 1+                except Full:+                    break+            with processes_management_lock:+                n_children_alive = sum(+                    p.is_alive() for p in list(processes.values())+                )++        # Release the queue's resources as soon as possible. Flag the feeder+        # thread for clean exit to avoid having the crash detection thread flag+        # the Executor as broken during the shutdown. This is safe as either:+        #  * We don't need to communicate with the workers anymore+        #  * There is nothing left in the Queue buffer except None sentinels+        mp.util.debug("closing call_queue")+        call_queue.close()++        mp.util.debug("joining processes")+        # If .join() is not called on the created processes then+        # some ctx.Queue methods may deadlock on Mac OS X.+        while processes:+            _, p = processes.popitem()+            p.join()+        mp.util.debug("queue management thread clean shutdown of worker "+                      "processes: {}".format(list(processes)))++    result_reader = result_queue._reader+    wakeup_reader = thread_wakeup._reader+    readers = [result_reader, wakeup_reader]++    while True:+        _add_call_item_to_queue(pending_work_items,+                                running_work_items,+                                work_ids_queue,+                                call_queue)+        # Wait for a result to be ready in the result_queue while checking+        # that all worker processes are still running, or for a wake up+        # signal send. The wake up signals come either from new tasks being+        # submitted, from the executor being shutdown/gc-ed, or from the+        # shutdown of the python interpreter.+        worker_sentinels = [p.sentinel for p in processes.values()]+        ready = wait(readers + worker_sentinels)++        broken = ("A process in the executor was terminated abruptly", None)+        if result_reader in ready:+            try:+                result_item = result_reader.recv()+                broken = None+            except BaseException as e:+                tb = getattr(e, "__traceback__", None)+                if tb is None:+                    _, _, tb = sys.exc_info()+                broken = ("A result has failed to un-serialize",+                          traceback.format_exception(type(e), e, tb))+        elif wakeup_reader in ready:+            broken = None+            result_item = None+        thread_wakeup.clear()+        if broken:+            msg, cause = broken+            # Mark the process pool broken so that submits fail right now.+            executor_flags.flag_as_broken(+                msg + ", the pool is not usable anymore.")+            bpe = BrokenProcessPool(+                msg + " while the future was running or pending.")+            if cause is not None:+                bpe.__cause__ = _RemoteTraceback(+                    "\n'''\n{}'''".format(''.join(cause)))++            # All futures in flight must be marked failed+            for work_id, work_item in pending_work_items.items():+                work_item.future.set_exception(bpe)+                # Delete references to object. See issue16284+                del work_item+            pending_work_items.clear()++            # Terminate remaining workers forcibly: the queues or their+            # locks may be in a dirty state and block forever.+            while processes:+                _, p = processes.popitem()+                mp.util.debug('terminate process {}'.format(p.name))+                try:+                    recursive_terminate(p)+                except ProcessLookupError:  # pragma: no cover+                    pass++            shutdown_all_workers()+            return+        if isinstance(result_item, int):+            # Clean shutdown of a worker using its PID, either on request+            # by the executor.shutdown method or by the timeout of the worker+            # itself: we should not mark the executor as broken.+            with processes_management_lock:+                p = processes.pop(result_item, None)++            # p can be None is the executor is concurrently shutting down.+            if p is not None:+                p._worker_exit_lock.release()+                p.join()+                del p++            # Make sure the executor have the right number of worker, even if a+            # worker timeout while some jobs were submitted. If some work is+            # pending or there is less processes than running items, we need to+            # start a new Process and raise a warning.+            n_pending = len(pending_work_items)+            n_running = len(running_work_items)+            if (n_pending - n_running > 0 or n_running > len(processes)):+                executor = executor_reference()+                if (executor is not None+                        and len(processes) < executor._max_workers):+                    warnings.warn(+                        "A worker stopped while some jobs were given to the "+                        "executor. This can be caused by a too short worker "+                        "timeout or by a memory leak.", UserWarning+                    )+                    executor._adjust_process_count()+                    executor = None++        elif result_item is not None:+            work_item = pending_work_items.pop(result_item.work_id, None)+            # work_item can be None if another process terminated+            if work_item is not None:+                if result_item.exception:+                    work_item.future.set_exception(result_item.exception)+                else:+                    work_item.future.set_result(result_item.result)+                # Delete references to object. See issue16284+                del work_item+                running_work_items.remove(result_item.work_id)+            # Delete reference to result_item+            del result_item++        # Check whether we should start shutting down.+        executor = executor_reference()+        # No more work items can be added if:+        #   - The interpreter is shutting down OR+        #   - The executor that owns this worker has been collected OR+        #   - The executor that owns this worker has been shutdown.+        if is_shutting_down():+            # bpo-33097: Make sure that the executor is flagged as shutting+            # down even if it is shutdown by the interpreter exiting.+            with executor_flags.shutdown_lock:+                executor_flags.shutdown = True+            if executor_flags.kill_workers:+                while pending_work_items:+                    _, work_item = pending_work_items.popitem()+                    work_item.future.set_exception(ShutdownExecutorError(+                        "The Executor was shutdown before this job could "+                        "complete."))+                    del work_item+                # Terminate remaining workers forcibly: the queues or their+                # locks may be in a dirty state and block forever.+                while processes:+                    _, p = processes.popitem()+                    recursive_terminate(p)+                shutdown_all_workers()+                return+            # Since no new work items can be added, it is safe to shutdown+            # this thread if there are no pending work items.+            if not pending_work_items:+                shutdown_all_workers()+                return+        elif executor_flags.broken:+            return+        executor = None+++_system_limits_checked = False+_system_limited = None+++def _check_system_limits():+    global _system_limits_checked, _system_limited+    if _system_limits_checked:+        if _system_limited:+            raise NotImplementedError(_system_limited)+    _system_limits_checked = True+    try:+        nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")+    except (AttributeError, ValueError):+        # sysconf not available or setting not available+        return+    if nsems_max == -1:+        # undetermined limit, assume that limit is determined+        # by available memory only+        return+    if nsems_max >= 256:+        # minimum number of semaphores available+        # according to POSIX+        return+    _system_limited = ("system provides too few semaphores (%d available, "+                       "256 necessary)" % nsems_max)+    raise NotImplementedError(_system_limited)+++def _chain_from_iterable_of_lists(iterable):+    """+    Specialized implementation of itertools.chain.from_iterable.+    Each item in *iterable* should be a list.  This function is+    careful not to keep references to yielded objects.+    """+    for element in iterable:+        element.reverse()+        while element:+            yield element.pop()+++def _check_max_depth(context):+    # Limit the maxmal recursion level+    global _CURRENT_DEPTH+    if context.get_start_method() == "fork" and _CURRENT_DEPTH > 0:+        raise LokyRecursionError(+            "Could not spawn extra nested processes at depth superior to "+            "MAX_DEPTH=1. It is not possible to increase this limit when "+            "using the 'fork' start method.")++    if 0 < MAX_DEPTH and _CURRENT_DEPTH + 1 > MAX_DEPTH:+        raise LokyRecursionError(+            "Could not spawn extra nested processes at depth superior to "+            "MAX_DEPTH={}. If this is intendend, you can change this limit "+            "with the LOKY_MAX_DEPTH environment variable.".format(MAX_DEPTH))+++class LokyRecursionError(RuntimeError):+    """Raised when a process try to spawn too many levels of nested processes.+    """+++class BrokenProcessPool(_BPPException):+    """+    Raised when a process in a ProcessPoolExecutor terminated abruptly+    while a future was in the running state.+    """+++# Alias for backward compat (for code written for loky 1.1.4 and earlier). Do+# not use in new code.+BrokenExecutor = BrokenProcessPool+++class ShutdownExecutorError(RuntimeError):++    """+    Raised when a ProcessPoolExecutor is shutdown while a future was in the+    running or pending state.+    """+++class ProcessPoolExecutor(_base.Executor):++    _at_exit = None++    def __init__(self, max_workers=None, job_reducers=None,+                 result_reducers=None, timeout=None, context=None,+                 initializer=None, initargs=()):+        """Initializes a new ProcessPoolExecutor instance.++        Args:+            max_workers: int, optional (default: cpu_count())+                The maximum number of processes that can be used to execute the+                given calls. If None or not given then as many worker processes+                will be created as the number of CPUs the current process+                can use.+            job_reducers, result_reducers: dict(type: reducer_func)+                Custom reducer for pickling the jobs and the results from the+                Executor. If only `job_reducers` is provided, `result_reducer`+                will use the same reducers+            timeout: int, optional (default: None)+                Idle workers exit after timeout seconds. If a new job is+                submitted after the timeout, the executor will start enough+                new Python processes to make sure the pool of workers is full.+            context: A multiprocessing context to launch the workers. This+                object should provide SimpleQueue, Queue and Process.+            initializer: An callable used to initialize worker processes.+            initargs: A tuple of arguments to pass to the initializer.+        """+        _check_system_limits()++        if max_workers is None:+            self._max_workers = cpu_count()+        else:+            if max_workers <= 0:+                raise ValueError("max_workers must be greater than 0")+            self._max_workers = max_workers++        if context is None:+            context = get_context()+        self._context = context++        if initializer is not None and not callable(initializer):+            raise TypeError("initializer must be a callable")+        self._initializer = initializer+        self._initargs = initargs++        _check_max_depth(self._context)++        if result_reducers is None:+            result_reducers = job_reducers++        # Timeout+        self._timeout = timeout++        # Internal variables of the ProcessPoolExecutor+        self._processes = {}+        self._queue_count = 0+        self._pending_work_items = {}+        self._running_work_items = []+        self._work_ids = queue.Queue()+        self._processes_management_lock = self._context.Lock()+        self._queue_management_thread = None++        # _ThreadWakeup is a communication channel used to interrupt the wait+        # of the main loop of queue_manager_thread from another thread (e.g.+        # when calling executor.submit or executor.shutdown). We do not use the+        # _result_queue to send the wakeup signal to the queue_manager_thread+        # as it could result in a deadlock if a worker process dies with the+        # _result_queue write lock still acquired.+        self._queue_management_thread_wakeup = _ThreadWakeup()++        # Flag to hold the state of the Executor. This permits to introspect+        # the Executor state even once it has been garbage collected.+        self._flags = _ExecutorFlags()++        # Finally setup the queues for interprocess communication+        self._setup_queues(job_reducers, result_reducers)++        mp.util.debug('ProcessPoolExecutor is setup')++    def _setup_queues(self, job_reducers, result_reducers, queue_size=None):+        # Make the call queue slightly larger than the number of processes to+        # prevent the worker processes from idling. But don't make it too big+        # because futures in the call queue cannot be cancelled.+        if queue_size is None:+            queue_size = 2 * self._max_workers + EXTRA_QUEUED_CALLS+        self._call_queue = _SafeQueue(+            max_size=queue_size, pending_work_items=self._pending_work_items,+            running_work_items=self._running_work_items,+            thread_wakeup=self._queue_management_thread_wakeup,+            reducers=job_reducers, ctx=self._context)+        # Killed worker processes can produce spurious "broken pipe"+        # tracebacks in the queue's own worker thread. But we detect killed+        # processes anyway, so silence the tracebacks.+        self._call_queue._ignore_epipe = True++        self._result_queue = SimpleQueue(reducers=result_reducers,+                                         ctx=self._context)++    def _start_queue_management_thread(self):+        if self._queue_management_thread is None:+            mp.util.debug('_start_queue_management_thread called')++            # When the executor gets garbarge collected, the weakref callback+            # will wake up the queue management thread so that it can terminate+            # if there is no pending work item.+            def weakref_cb(_,+                           thread_wakeup=self._queue_management_thread_wakeup):+                mp.util.debug('Executor collected: triggering callback for'+                              ' QueueManager wakeup')+                thread_wakeup.wakeup()++            # Start the processes so that their sentinels are known.+            self._queue_management_thread = threading.Thread(+                target=_queue_management_worker,+                args=(weakref.ref(self, weakref_cb),+                      self._flags,+                      self._processes,+                      self._pending_work_items,+                      self._running_work_items,+                      self._work_ids,+                      self._call_queue,+                      self._result_queue,+                      self._queue_management_thread_wakeup,+                      self._processes_management_lock),+                name="QueueManagerThread")+            self._queue_management_thread.daemon = True+            self._queue_management_thread.start()++            # register this executor in a mechanism that ensures it will wakeup+            # when the interpreter is exiting.+            _threads_wakeups[self._queue_management_thread] = \+                self._queue_management_thread_wakeup++            global process_pool_executor_at_exit+            if process_pool_executor_at_exit is None:+                # Ensure that the _python_exit function will be called before+                # the multiprocessing.Queue._close finalizers which have an+                # exitpriority of 10.+                process_pool_executor_at_exit = mp.util.Finalize(+                    None, _python_exit, exitpriority=20)++    def _adjust_process_count(self):+        for _ in range(len(self._processes), self._max_workers):+            worker_exit_lock = self._context.BoundedSemaphore(1)+            worker_exit_lock.acquire()+            p = self._context.Process(+                target=_process_worker,+                args=(self._call_queue,+                      self._result_queue,+                      self._initializer,+                      self._initargs,+                      self._processes_management_lock,+                      self._timeout,+                      worker_exit_lock,+                      _CURRENT_DEPTH + 1))+            p._worker_exit_lock = worker_exit_lock+            p.start()+            self._processes[p.pid] = p+        mp.util.debug('Adjust process count : {}'.format(self._processes))++    def _ensure_executor_running(self):+        """ensures all workers and management thread are running+        """+        with self._processes_management_lock:+            if len(self._processes) != self._max_workers:+                self._adjust_process_count()+            self._start_queue_management_thread()++    def submit(self, fn, *args, **kwargs):+        with self._flags.shutdown_lock:+            if self._flags.broken:+                raise BrokenProcessPool(self._flags.broken)+            if self._flags.shutdown:+                raise ShutdownExecutorError(+                    'cannot schedule new futures after shutdown')++            # Cannot submit a new calls once the interpreter is shutting down.+            # This check avoids spawning new processes at exit.+            if _global_shutdown:+                raise RuntimeError('cannot schedule new futures after '+                                   'interpreter shutdown')++            f = _base.Future()+            w = _WorkItem(f, fn, args, kwargs)++            self._pending_work_items[self._queue_count] = w+            self._work_ids.put(self._queue_count)+            self._queue_count += 1+            # Wake up queue management thread+            self._queue_management_thread_wakeup.wakeup()++            self._ensure_executor_running()+            return f+    submit.__doc__ = _base.Executor.submit.__doc__++    def map(self, fn, *iterables, **kwargs):+        """Returns an iterator equivalent to map(fn, iter).++        Args:+            fn: A callable that will take as many arguments as there are+                passed iterables.+            timeout: The maximum number of seconds to wait. If None, then there+                is no limit on the wait time.+            chunksize: If greater than one, the iterables will be chopped into+                chunks of size chunksize and submitted to the process pool.+                If set to one, the items in the list will be sent one at a+                time.++        Returns:+            An iterator equivalent to: map(func, *iterables) but the calls may+            be evaluated out-of-order.++        Raises:+            TimeoutError: If the entire result iterator could not be generated+                before the given timeout.+            Exception: If fn(*args) raises for any values.+        """+        timeout = kwargs.get('timeout', None)+        chunksize = kwargs.get('chunksize', 1)+        if chunksize < 1:+            raise ValueError("chunksize must be >= 1.")++        results = super(ProcessPoolExecutor, self).map(+            partial(_process_chunk, fn), _get_chunks(chunksize, *iterables),+            timeout=timeout)+        return _chain_from_iterable_of_lists(results)++    def shutdown(self, wait=True, kill_workers=False):+        mp.util.debug('shutting down executor %s' % self)++        self._flags.flag_as_shutting_down(kill_workers)+        qmt = self._queue_management_thread+        qmtw = self._queue_management_thread_wakeup+        if qmt:+            self._queue_management_thread = None+            if qmtw:+                self._queue_management_thread_wakeup = None+            # Wake up queue management thread+            if qmtw is not None:+                try:+                    qmtw.wakeup()+                except OSError:+                    # Can happen in case of concurrent calls to shutdown.+                    pass+            if wait:+                qmt.join()++        cq = self._call_queue+        if cq:+            self._call_queue = None+            cq.close()+            if wait:+                cq.join_thread()+        self._result_queue = None+        self._processes_management_lock = None++        if qmtw:+            try:+                qmtw.close()+            except OSError:+                # Can happen in case of concurrent calls to shutdown.+                pass+    shutdown.__doc__ = _base.Executor.shutdown.__doc__
+ test/files/mypy.py view
@@ -0,0 +1,4017 @@+"""Mypy type checker."""++import itertools+import fnmatch+from contextlib import contextmanager++from typing import (+    Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Any+)++from mypy.errors import Errors, report_internal_error+from mypy.nodes import (+    SymbolTable, Statement, MypyFile, Var, Expression, Lvalue, Node,+    OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo,+    ClassDef, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr,+    TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt,+    WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt,+    RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr,+    UnicodeExpr, OpExpr, UnaryExpr, LambdaExpr, TempNode, SymbolTableNode,+    Context, Decorator, PrintStmt, BreakStmt, PassStmt, ContinueStmt,+    ComparisonExpr, StarExpr, EllipsisExpr, RefExpr, PromoteExpr,+    Import, ImportFrom, ImportAll, ImportBase, TypeAlias,+    ARG_POS, ARG_STAR, LITERAL_TYPE, MDEF, GDEF,+    CONTRAVARIANT, COVARIANT, INVARIANT,+)+from mypy import nodes+from mypy.literals import literal, literal_hash+from mypy.typeanal import has_any_from_unimported_type, check_for_explicit_any+from mypy.types import (+    Type, AnyType, CallableType, FunctionLike, Overloaded, TupleType, TypedDictType,+    Instance, NoneTyp, strip_type, TypeType, TypeOfAny,+    UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef,+    true_only, false_only, function_type, is_named_instance, union_items,+)+from mypy.sametypes import is_same_type, is_same_types+from mypy.messages import MessageBuilder, make_inferred_type_note+import mypy.checkexpr+from mypy.checkmember import (+    map_type_from_supertype, bind_self, erase_to_bound, type_object_type,+    analyze_descriptor_access+)+from mypy import messages+from mypy.subtypes import (+    is_subtype, is_equivalent, is_proper_subtype, is_more_precise,+    restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_compatible,+    unify_generic_callable, find_member+)+from mypy.constraints import SUPERTYPE_OF+from mypy.maptype import map_instance_to_supertype+from mypy.typevars import fill_typevars, has_no_typevars+from mypy.semanal import set_callable_name, refers_to_fullname, calculate_mro+from mypy.erasetype import erase_typevars+from mypy.expandtype import expand_type, expand_type_by_instance+from mypy.visitor import NodeVisitor+from mypy.join import join_types+from mypy.treetransform import TransformVisitor+from mypy.binder import ConditionalTypeBinder, get_declaration+from mypy.meet import is_overlapping_types, is_partially_overlapping_types+from mypy.options import Options+from mypy.plugin import Plugin, CheckerPluginInterface+from mypy.sharedparse import BINARY_MAGIC_METHODS+from mypy.scope import Scope++from mypy import experiments+++T = TypeVar('T')++DEFAULT_LAST_PASS = 1  # Pass numbers start at 0+++# A node which is postponed to be processed during the next pass.+# This is used for both batch mode and fine-grained incremental mode.+DeferredNode = NamedTuple(+    'DeferredNode',+    [+        # In batch mode only FuncDef and LambdaExpr are supported+        ('node', Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef]),+        ('context_type_name', Optional[str]),  # Name of the surrounding class (for error messages)+        ('active_typeinfo', Optional[TypeInfo]),  # And its TypeInfo (for semantic analysis+                                                  # self type handling)+    ])+++# Data structure returned by find_isinstance_check representing+# information learned from the truth or falsehood of a condition.  The+# dict maps nodes representing expressions like 'a[0].x' to their+# refined types under the assumption that the condition has a+# particular truth value. A value of None means that the condition can+# never have that truth value.++# NB: The keys of this dict are nodes in the original source program,+# which are compared by reference equality--effectively, being *the+# same* expression of the program, not just two identical expressions+# (such as two references to the same variable). TODO: it would+# probably be better to have the dict keyed by the nodes' literal_hash+# field instead.++TypeMap = Optional[Dict[Expression, Type]]++# An object that represents either a precise type or a type with an upper bound;+# it is important for correct type inference with isinstance.+TypeRange = NamedTuple(+    'TypeRange',+    [+        ('item', Type),+        ('is_upper_bound', bool),  # False => precise type+    ])++# Keeps track of partial types in a single scope. In fine-grained incremental+# mode partial types initially defined at the top level cannot be completed in+# a function, and we use the 'is_function' attribute to enforce this.+PartialTypeScope = NamedTuple('PartialTypeScope', [('map', Dict[Var, Context]),+                                                   ('is_function', bool)])+++class TypeChecker(NodeVisitor[None], CheckerPluginInterface):+    """Mypy type checker.++    Type check mypy source files that have been semantically analyzed.++    You must create a separate instance for each source file.+    """++    # Are we type checking a stub?+    is_stub = False+    # Error message reporter+    errors = None  # type: Errors+    # Utility for generating messages+    msg = None  # type: MessageBuilder+    # Types of type checked nodes+    type_map = None  # type: Dict[Expression, Type]++    # Helper for managing conditional types+    binder = None  # type: ConditionalTypeBinder+    # Helper for type checking expressions+    expr_checker = None  # type: mypy.checkexpr.ExpressionChecker++    tscope = None  # type: Scope+    scope = None  # type: CheckerScope+    # Stack of function return types+    return_types = None  # type: List[Type]+    # Flags; true for dynamically typed functions+    dynamic_funcs = None  # type: List[bool]+    # Stack of collections of variables with partial types+    partial_types = None  # type: List[PartialTypeScope]+    # Vars for which partial type errors are already reported+    # (to avoid logically duplicate errors with different error context).+    partial_reported = None  # type: Set[Var]+    globals = None  # type: SymbolTable+    modules = None  # type: Dict[str, MypyFile]+    # Nodes that couldn't be checked because some types weren't available. We'll run+    # another pass and try these again.+    deferred_nodes = None  # type: List[DeferredNode]+    # Type checking pass number (0 = first pass)+    pass_num = 0+    # Last pass number to take+    last_pass = DEFAULT_LAST_PASS+    # Have we deferred the current function? If yes, don't infer additional+    # types during this pass within the function.+    current_node_deferred = False+    # Is this file a typeshed stub?+    is_typeshed_stub = False+    # Should strict Optional-related errors be suppressed in this file?+    suppress_none_errors = False  # TODO: Get it from options instead+    options = None  # type: Options+    # Used for collecting inferred attribute types so that they can be checked+    # for consistency.+    inferred_attribute_types = None  # type: Optional[Dict[Var, Type]]+    # Don't infer partial None types if we are processing assignment from Union+    no_partial_types = False  # type: bool++    # The set of all dependencies (suppressed or not) that this module accesses, either+    # directly or indirectly.+    module_refs = None  # type: Set[str]++    # Plugin that provides special type checking rules for specific library+    # functions such as open(), etc.+    plugin = None  # type: Plugin++    def __init__(self, errors: Errors, modules: Dict[str, MypyFile], options: Options,+                 tree: MypyFile, path: str, plugin: Plugin) -> None:+        """Construct a type checker.++        Use errors to report type check errors.+        """+        self.errors = errors+        self.modules = modules+        self.options = options+        self.tree = tree+        self.path = path+        self.msg = MessageBuilder(errors, modules)+        self.plugin = plugin+        self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg, self.plugin)+        self.tscope = Scope()+        self.scope = CheckerScope(tree)+        self.binder = ConditionalTypeBinder()+        self.globals = tree.names+        self.return_types = []+        self.dynamic_funcs = []+        self.partial_types = []+        self.partial_reported = set()+        self.deferred_nodes = []+        self.type_map = {}+        self.module_refs = set()+        self.pass_num = 0+        self.current_node_deferred = False+        self.is_stub = tree.is_stub+        self.is_typeshed_stub = errors.is_typeshed_file(path)+        self.inferred_attribute_types = None+        if options.strict_optional_whitelist is None:+            self.suppress_none_errors = not options.show_none_errors+        else:+            self.suppress_none_errors = not any(fnmatch.fnmatch(path, pattern)+                                                for pattern+                                                in options.strict_optional_whitelist)+        # If True, process function definitions. If False, don't. This is used+        # for processing module top levels in fine-grained incremental mode.+        self.recurse_into_functions = True++    def reset(self) -> None:+        """Cleanup stale state that might be left over from a typechecking run.++        This allows us to reuse TypeChecker objects in fine-grained+        incremental mode.+        """+        # TODO: verify this is still actually worth it over creating new checkers+        self.partial_reported.clear()+        self.module_refs.clear()+        self.binder = ConditionalTypeBinder()+        self.type_map.clear()++        assert self.inferred_attribute_types is None+        assert self.partial_types == []+        assert self.deferred_nodes == []+        assert len(self.scope.stack) == 1+        assert self.partial_types == []++    def check_first_pass(self) -> None:+        """Type check the entire file, but defer functions with unresolved references.++        Unresolved references are forward references to variables+        whose types haven't been inferred yet.  They may occur later+        in the same file or in a different file that's being processed+        later (usually due to an import cycle).++        Deferred functions will be processed by check_second_pass().+        """+        self.recurse_into_functions = True+        with experiments.strict_optional_set(self.options.strict_optional):+            self.errors.set_file(self.path, self.tree.fullname(), scope=self.tscope)+            self.tscope.enter_file(self.tree.fullname())+            with self.enter_partial_types():+                with self.binder.top_frame_context():+                    for d in self.tree.defs:+                        self.accept(d)++            assert not self.current_node_deferred++            all_ = self.globals.get('__all__')+            if all_ is not None and all_.type is not None:+                all_node = all_.node+                assert all_node is not None+                seq_str = self.named_generic_type('typing.Sequence',+                                                [self.named_type('builtins.str')])+                if self.options.python_version[0] < 3:+                    seq_str = self.named_generic_type('typing.Sequence',+                                                    [self.named_type('builtins.unicode')])+                if not is_subtype(all_.type, seq_str):+                    str_seq_s, all_s = self.msg.format_distinctly(seq_str, all_.type)+                    self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s),+                            all_node)++            self.tscope.leave()++    def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool:+        """Run second or following pass of type checking.++        This goes through deferred nodes, returning True if there were any.+        """+        self.recurse_into_functions = True+        with experiments.strict_optional_set(self.options.strict_optional):+            if not todo and not self.deferred_nodes:+                return False+            self.errors.set_file(self.path, self.tree.fullname(), scope=self.tscope)+            self.tscope.enter_file(self.tree.fullname())+            self.pass_num += 1+            if not todo:+                todo = self.deferred_nodes+            else:+                assert not self.deferred_nodes+            self.deferred_nodes = []+            done = set()  # type: Set[Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef]]+            for node, type_name, active_typeinfo in todo:+                if node in done:+                    continue+                # This is useful for debugging:+                # print("XXX in pass %d, class %s, function %s" %+                #       (self.pass_num, type_name, node.fullname() or node.name()))+                done.add(node)+                with self.tscope.class_scope(active_typeinfo) if active_typeinfo else nothing():+                    with self.scope.push_class(active_typeinfo) if active_typeinfo else nothing():+                        self.check_partial(node)+            self.tscope.leave()+            return True++    def check_partial(self, node: Union[FuncDef,+                                        LambdaExpr,+                                        MypyFile,+                                        OverloadedFuncDef]) -> None:+        if isinstance(node, MypyFile):+            self.check_top_level(node)+        else:+            self.recurse_into_functions = True+            if isinstance(node, LambdaExpr):+                self.expr_checker.accept(node)+            else:+                self.accept(node)++    def check_top_level(self, node: MypyFile) -> None:+        """Check only the top-level of a module, skipping function definitions."""+        self.recurse_into_functions = False+        with self.enter_partial_types():+            with self.binder.top_frame_context():+                for d in node.defs:+                    d.accept(self)++        assert not self.current_node_deferred+        # TODO: Handle __all__++    def handle_cannot_determine_type(self, name: str, context: Context) -> None:+        node = self.scope.top_non_lambda_function()+        if self.pass_num < self.last_pass and isinstance(node, FuncDef):+            # Don't report an error yet. Just defer. Note that we don't defer+            # lambdas because they are coupled to the surrounding function+            # through the binder and the inferred type of the lambda, so it+            # would get messy.+            if self.errors.type_name:+                type_name = self.errors.type_name[-1]+            else:+                type_name = None+            # Shouldn't we freeze the entire scope?+            enclosing_class = self.scope.enclosing_class()+            self.deferred_nodes.append(DeferredNode(node, type_name, enclosing_class))+            # Set a marker so that we won't infer additional types in this+            # function. Any inferred types could be bogus, because there's at+            # least one type that we don't know.+            self.current_node_deferred = True+        else:+            self.msg.cannot_determine_type(name, context)++    def accept(self, stmt: Statement) -> None:+        """Type check a node in the given type context."""+        try:+            stmt.accept(self)+        except Exception as err:+            report_internal_error(err, self.errors.file, stmt.line, self.errors, self.options)++    def accept_loop(self, body: Statement, else_body: Optional[Statement] = None, *,+                    exit_condition: Optional[Expression] = None) -> None:+        """Repeatedly type check a loop body until the frame doesn't change.+        If exit_condition is set, assume it must be False on exit from the loop.++        Then check the else_body.+        """+        # The outer frame accumulates the results of all iterations+        with self.binder.frame_context(can_skip=False):+            while True:+                with self.binder.frame_context(can_skip=True,+                                               break_frame=2, continue_frame=1):+                    self.accept(body)+                if not self.binder.last_pop_changed:+                    break+            if exit_condition:+                _, else_map = self.find_isinstance_check(exit_condition)+                self.push_type_map(else_map)+            if else_body:+                self.accept(else_body)++    #+    # Definitions+    #++    def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:+        if not self.recurse_into_functions:+            return+        with self.tscope.function_scope(defn):+            self._visit_overloaded_func_def(defn)++    def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:+        num_abstract = 0+        if not defn.items:+            # In this case we have already complained about none of these being+            # valid overloads.+            return None+        if len(defn.items) == 1:+            self.fail('Single overload definition, multiple required', defn)++        if defn.is_property:+            # HACK: Infer the type of the property.+            self.visit_decorator(cast(Decorator, defn.items[0]))+        for fdef in defn.items:+            assert isinstance(fdef, Decorator)+            self.check_func_item(fdef.func, name=fdef.func.name())+            if fdef.func.is_abstract:+                num_abstract += 1+        if num_abstract not in (0, len(defn.items)):+            self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn)+        if defn.impl:+            defn.impl.accept(self)+        if defn.info:+            self.check_method_override(defn)+            self.check_inplace_operator_method(defn)+        if not defn.is_property:+            self.check_overlapping_overloads(defn)+        return None++    def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None:+        # At this point we should have set the impl already, and all remaining+        # items are decorators++        # Compute some info about the implementation (if it exists) for use below+        impl_type = None  # type: Optional[CallableType]+        if defn.impl:+            if isinstance(defn.impl, FuncDef):+                inner_type = defn.impl.type+            elif isinstance(defn.impl, Decorator):+                inner_type = defn.impl.var.type+            else:+                assert False, "Impl isn't the right type"++            # This can happen if we've got an overload with a different+            # decorator or if the implementation is untyped -- we gave up on the types.+            if inner_type is not None and not isinstance(inner_type, AnyType):+                assert isinstance(inner_type, CallableType)+                impl_type = inner_type++        is_descriptor_get = defn.info and defn.name() == "__get__"+        for i, item in enumerate(defn.items):+            # TODO overloads involving decorators+            assert isinstance(item, Decorator)+            sig1 = self.function_type(item.func)+            assert isinstance(sig1, CallableType)++            for j, item2 in enumerate(defn.items[i + 1:]):+                assert isinstance(item2, Decorator)+                sig2 = self.function_type(item2.func)+                assert isinstance(sig2, CallableType)++                if not are_argument_counts_overlapping(sig1, sig2):+                    continue++                if overload_can_never_match(sig1, sig2):+                    self.msg.overloaded_signature_will_never_match(+                        i + 1, i + j + 2, item2.func)+                elif not is_descriptor_get:+                    # Note: we force mypy to check overload signatures in strict-optional mode+                    # so we don't incorrectly report errors when a user tries typing an overload+                    # that happens to have a 'if the argument is None' fallback.+                    #+                    # For example, the following is fine in strict-optional mode but would throw+                    # the unsafe overlap error when strict-optional is disabled:+                    #+                    #     @overload+                    #     def foo(x: None) -> int: ...+                    #     @overload+                    #     def foo(x: str) -> str: ...+                    #+                    # See Python 2's map function for a concrete example of this kind of overload.+                    with experiments.strict_optional_set(True):+                        if is_unsafe_overlapping_overload_signatures(sig1, sig2):+                            self.msg.overloaded_signatures_overlap(+                                i + 1, i + j + 2, item.func)++            if impl_type is not None:+                assert defn.impl is not None++                # We perform a unification step that's very similar to what+                # 'is_callable_compatible' would have done if we had set+                # 'unify_generics' to True -- the only difference is that+                # we check and see if the impl_type's return value is a+                # *supertype* of the overload alternative, not a *subtype*.+                #+                # This is to match the direction the implementation's return+                # needs to be compatible in.+                if impl_type.variables:+                    impl = unify_generic_callable(impl_type, sig1,+                                                  ignore_return=False,+                                                  return_constraint_direction=SUPERTYPE_OF)+                    if impl is None:+                        self.msg.overloaded_signatures_typevar_specific(i + 1, defn.impl)+                        continue+                else:+                    impl = impl_type++                # Is the overload alternative's arguments subtypes of the implementation's?+                if not is_callable_compatible(impl, sig1,+                                              is_compat=is_subtype,+                                              ignore_return=True):+                    self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl)++                # Is the overload alternative's return type a subtype of the implementation's?+                if not is_subtype(sig1.ret_type, impl.ret_type):+                    self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl)++    # Here's the scoop about generators and coroutines.+    #+    # There are two kinds of generators: classic generators (functions+    # with `yield` or `yield from` in the body) and coroutines+    # (functions declared with `async def`).  The latter are specified+    # in PEP 492 and only available in Python >= 3.5.+    #+    # Classic generators can be parameterized with three types:+    # - ty is the Yield type (the type of y in `yield y`)+    # - tc is the type reCeived by yield (the type of c in `c = yield`).+    # - tr is the Return type (the type of r in `return r`)+    #+    # A classic generator must define a return type that's either+    # `Generator[ty, tc, tr]`, Iterator[ty], or Iterable[ty] (or+    # object or Any).  If tc/tr are not given, both are None.+    #+    # A coroutine must define a return type corresponding to tr; the+    # other two are unconstrained.  The "external" return type (seen+    # by the caller) is Awaitable[tr].+    #+    # In addition, there's the synthetic type AwaitableGenerator: it+    # inherits from both Awaitable and Generator and can be used both+    # in `yield from` and in `await`.  This type is set automatically+    # for functions decorated with `@types.coroutine` or+    # `@asyncio.coroutine`.  Its single parameter corresponds to tr.+    #+    # PEP 525 adds a new type, the asynchronous generator, which was+    # first released in Python 3.6. Async generators are `async def`+    # functions that can also `yield` values. They can be parameterized+    # with two types, ty and tc, because they cannot return a value.+    #+    # There are several useful methods, each taking a type t and a+    # flag c indicating whether it's for a generator or coroutine:+    #+    # - is_generator_return_type(t, c) returns whether t is a Generator,+    #   Iterator, Iterable (if not c), or Awaitable (if c), or+    #   AwaitableGenerator (regardless of c).+    # - is_async_generator_return_type(t) returns whether t is an+    #   AsyncGenerator.+    # - get_generator_yield_type(t, c) returns ty.+    # - get_generator_receive_type(t, c) returns tc.+    # - get_generator_return_type(t, c) returns tr.++    def is_generator_return_type(self, typ: Type, is_coroutine: bool) -> bool:+        """Is `typ` a valid type for a generator/coroutine?++        True if `typ` is a *supertype* of Generator or Awaitable.+        Also true it it's *exactly* AwaitableGenerator (modulo type parameters).+        """+        if is_coroutine:+            # This means we're in Python 3.5 or later.+            at = self.named_generic_type('typing.Awaitable', [AnyType(TypeOfAny.special_form)])+            if is_subtype(at, typ):+                return True+        else:+            any_type = AnyType(TypeOfAny.special_form)+            gt = self.named_generic_type('typing.Generator', [any_type, any_type, any_type])+            if is_subtype(gt, typ):+                return True+        return isinstance(typ, Instance) and typ.type.fullname() == 'typing.AwaitableGenerator'++    def is_async_generator_return_type(self, typ: Type) -> bool:+        """Is `typ` a valid type for an async generator?++        True if `typ` is a supertype of AsyncGenerator.+        """+        try:+            any_type = AnyType(TypeOfAny.special_form)+            agt = self.named_generic_type('typing.AsyncGenerator', [any_type, any_type])+        except KeyError:+            # we're running on a version of typing that doesn't have AsyncGenerator yet+            return False+        return is_subtype(agt, typ)++    def get_generator_yield_type(self, return_type: Type, is_coroutine: bool) -> Type:+        """Given the declared return type of a generator (t), return the type it yields (ty)."""+        if isinstance(return_type, AnyType):+            return AnyType(TypeOfAny.from_another_any, source_any=return_type)+        elif (not self.is_generator_return_type(return_type, is_coroutine)+                and not self.is_async_generator_return_type(return_type)):+            # If the function doesn't have a proper Generator (or+            # Awaitable) return type, anything is permissible.+            return AnyType(TypeOfAny.from_error)+        elif not isinstance(return_type, Instance):+            # Same as above, but written as a separate branch so the typechecker can understand.+            return AnyType(TypeOfAny.from_error)+        elif return_type.type.fullname() == 'typing.Awaitable':+            # Awaitable: ty is Any.+            return AnyType(TypeOfAny.special_form)+        elif return_type.args:+            # AwaitableGenerator, Generator, AsyncGenerator, Iterator, or Iterable; ty is args[0].+            ret_type = return_type.args[0]+            # TODO not best fix, better have dedicated yield token+            return ret_type+        else:+            # If the function's declared supertype of Generator has no type+            # parameters (i.e. is `object`), then the yielded values can't+            # be accessed so any type is acceptable.  IOW, ty is Any.+            # (However, see https://github.com/python/mypy/issues/1933)+            return AnyType(TypeOfAny.special_form)++    def get_generator_receive_type(self, return_type: Type, is_coroutine: bool) -> Type:+        """Given a declared generator return type (t), return the type its yield receives (tc)."""+        if isinstance(return_type, AnyType):+            return AnyType(TypeOfAny.from_another_any, source_any=return_type)+        elif (not self.is_generator_return_type(return_type, is_coroutine)+                and not self.is_async_generator_return_type(return_type)):+            # If the function doesn't have a proper Generator (or+            # Awaitable) return type, anything is permissible.+            return AnyType(TypeOfAny.from_error)+        elif not isinstance(return_type, Instance):+            # Same as above, but written as a separate branch so the typechecker can understand.+            return AnyType(TypeOfAny.from_error)+        elif return_type.type.fullname() == 'typing.Awaitable':+            # Awaitable, AwaitableGenerator: tc is Any.+            return AnyType(TypeOfAny.special_form)+        elif (return_type.type.fullname() in ('typing.Generator', 'typing.AwaitableGenerator')+              and len(return_type.args) >= 3):+            # Generator: tc is args[1].+            return return_type.args[1]+        elif return_type.type.fullname() == 'typing.AsyncGenerator' and len(return_type.args) >= 2:+            return return_type.args[1]+        else:+            # `return_type` is a supertype of Generator, so callers won't be able to send it+            # values.  IOW, tc is None.+            return NoneTyp()++    def get_coroutine_return_type(self, return_type: Type) -> Type:+        if isinstance(return_type, AnyType):+            return AnyType(TypeOfAny.from_another_any, source_any=return_type)+        assert isinstance(return_type, Instance), "Should only be called on coroutine functions."+        # Note: return type is the 3rd type parameter of Coroutine.+        return return_type.args[2]++    def get_generator_return_type(self, return_type: Type, is_coroutine: bool) -> Type:+        """Given the declared return type of a generator (t), return the type it returns (tr)."""+        if isinstance(return_type, AnyType):+            return AnyType(TypeOfAny.from_another_any, source_any=return_type)+        elif not self.is_generator_return_type(return_type, is_coroutine):+            # If the function doesn't have a proper Generator (or+            # Awaitable) return type, anything is permissible.+            return AnyType(TypeOfAny.from_error)+        elif not isinstance(return_type, Instance):+            # Same as above, but written as a separate branch so the typechecker can understand.+            return AnyType(TypeOfAny.from_error)+        elif return_type.type.fullname() == 'typing.Awaitable' and len(return_type.args) == 1:+            # Awaitable: tr is args[0].+            return return_type.args[0]+        elif (return_type.type.fullname() in ('typing.Generator', 'typing.AwaitableGenerator')+              and len(return_type.args) >= 3):+            # AwaitableGenerator, Generator: tr is args[2].+            return return_type.args[2]+        else:+            # Supertype of Generator (Iterator, Iterable, object): tr is any.+            return AnyType(TypeOfAny.special_form)++    def visit_func_def(self, defn: FuncDef) -> None:+        if not self.recurse_into_functions:+            return+        with self.tscope.function_scope(defn):+            self._visit_func_def(defn)++    def _visit_func_def(self, defn: FuncDef) -> None:+        """Type check a function definition."""+        self.check_func_item(defn, name=defn.name())+        if defn.info:+            if not defn.is_dynamic() and not defn.is_overload and not defn.is_decorated:+                # If the definition is the implementation for an+                # overload, the legality of the override has already+                # been typechecked, and decorated methods will be+                # checked when the decorator is.+                self.check_method_override(defn)+            self.check_inplace_operator_method(defn)+        if defn.original_def:+            # Override previous definition.+            new_type = self.function_type(defn)+            if isinstance(defn.original_def, FuncDef):+                # Function definition overrides function definition.+                if not is_same_type(new_type, self.function_type(defn.original_def)):+                    self.msg.incompatible_conditional_function_def(defn)+            else:+                # Function definition overrides a variable initialized via assignment or a+                # decorated function.+                orig_type = defn.original_def.type+                if orig_type is None:+                    # XXX This can be None, as happens in+                    # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse+                    self.msg.note("Internal mypy error checking function redefinition", defn)+                    return+                if isinstance(orig_type, PartialType):+                    if orig_type.type is None:+                        # Ah this is a partial type. Give it the type of the function.+                        orig_def = defn.original_def+                        if isinstance(orig_def, Decorator):+                            var = orig_def.var+                        else:+                            var = orig_def+                        partial_types = self.find_partial_types(var)+                        if partial_types is not None:+                            var.type = new_type+                            del partial_types[var]+                    else:+                        # Trying to redefine something like partial empty list as function.+                        self.fail(messages.INCOMPATIBLE_REDEFINITION, defn)+                else:+                    # TODO: Update conditional type binder.+                    self.check_subtype(new_type, orig_type, defn,+                                       messages.INCOMPATIBLE_REDEFINITION,+                                       'redefinition with type',+                                       'original type')++    def check_func_item(self, defn: FuncItem,+                        type_override: Optional[CallableType] = None,+                        name: Optional[str] = None) -> None:+        """Type check a function.++        If type_override is provided, use it as the function type.+        """+        self.dynamic_funcs.append(defn.is_dynamic() and not type_override)++        with self.enter_partial_types(is_function=True):+            typ = self.function_type(defn)+            if type_override:+                typ = type_override.copy_modified(line=typ.line, column=typ.column)+            if isinstance(typ, CallableType):+                with self.enter_attribute_inference_context():+                    self.check_func_def(defn, typ, name)+            else:+                raise RuntimeError('Not supported')++        self.dynamic_funcs.pop()+        self.current_node_deferred = False++    @contextmanager+    def enter_attribute_inference_context(self) -> Iterator[None]:+        old_types = self.inferred_attribute_types+        self.inferred_attribute_types = {}+        yield None+        self.inferred_attribute_types = old_types++    def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str]) -> None:+        """Type check a function definition."""+        # Expand type variables with value restrictions to ordinary types.+        for item, typ in self.expand_typevars(defn, typ):+            old_binder = self.binder+            self.binder = ConditionalTypeBinder()+            with self.binder.top_frame_context():+                defn.expanded.append(item)++                # We may be checking a function definition or an anonymous+                # function. In the first case, set up another reference with the+                # precise type.+                if isinstance(item, FuncDef):+                    fdef = item+                    # Check if __init__ has an invalid, non-None return type.+                    if (fdef.info and fdef.name() in ('__init__', '__init_subclass__') and+                            not isinstance(typ.ret_type, NoneTyp) and+                            not self.dynamic_funcs[-1]):+                        self.fail(messages.MUST_HAVE_NONE_RETURN_TYPE.format(fdef.name()),+                                  item)++                    self.check_for_missing_annotations(fdef)+                    if self.options.disallow_any_unimported:+                        if fdef.type and isinstance(fdef.type, CallableType):+                            ret_type = fdef.type.ret_type+                            if has_any_from_unimported_type(ret_type):+                                self.msg.unimported_type_becomes_any("Return type", ret_type, fdef)+                            for idx, arg_type in enumerate(fdef.type.arg_types):+                                if has_any_from_unimported_type(arg_type):+                                    prefix = "Argument {} to \"{}\"".format(idx + 1, fdef.name())+                                    self.msg.unimported_type_becomes_any(prefix, arg_type, fdef)+                    check_for_explicit_any(fdef.type, self.options, self.is_typeshed_stub,+                                           self.msg, context=fdef)++                if name:  # Special method names+                    if defn.info and self.is_reverse_op_method(name):+                        self.check_reverse_op_method(item, typ, name, defn)+                    elif name in ('__getattr__', '__getattribute__'):+                        self.check_getattr_method(typ, defn, name)+                    elif name == '__setattr__':+                        self.check_setattr_method(typ, defn)++                # Refuse contravariant return type variable+                if isinstance(typ.ret_type, TypeVarType):+                    if typ.ret_type.variance == CONTRAVARIANT:+                        self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT,+                             typ.ret_type)++                # Check that Generator functions have the appropriate return type.+                if defn.is_generator:+                    if defn.is_async_generator:+                        if not self.is_async_generator_return_type(typ.ret_type):+                            self.fail(messages.INVALID_RETURN_TYPE_FOR_ASYNC_GENERATOR, typ)+                    else:+                        if not self.is_generator_return_type(typ.ret_type, defn.is_coroutine):+                            self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ)++                    # Python 2 generators aren't allowed to return values.+                    if (self.options.python_version[0] == 2 and+                            isinstance(typ.ret_type, Instance) and+                            typ.ret_type.type.fullname() == 'typing.Generator'):+                        if not isinstance(typ.ret_type.args[2], (NoneTyp, AnyType)):+                            self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ)++                # Fix the type if decorated with `@types.coroutine` or `@asyncio.coroutine`.+                if defn.is_awaitable_coroutine:+                    # Update the return type to AwaitableGenerator.+                    # (This doesn't exist in typing.py, only in typing.pyi.)+                    t = typ.ret_type+                    c = defn.is_coroutine+                    ty = self.get_generator_yield_type(t, c)+                    tc = self.get_generator_receive_type(t, c)+                    if c:+                        tr = self.get_coroutine_return_type(t)+                    else:+                        tr = self.get_generator_return_type(t, c)+                    ret_type = self.named_generic_type('typing.AwaitableGenerator',+                                                       [ty, tc, tr, t])+                    typ = typ.copy_modified(ret_type=ret_type)+                    defn.type = typ++                # Push return type.+                self.return_types.append(typ.ret_type)++                # Store argument types.+                for i in range(len(typ.arg_types)):+                    arg_type = typ.arg_types[i]++                    ref_type = self.scope.active_self_type()  # type: Optional[Type]+                    if (isinstance(defn, FuncDef) and ref_type is not None and i == 0+                            and not defn.is_static+                            and typ.arg_kinds[0] not in [nodes.ARG_STAR, nodes.ARG_STAR2]):+                        isclass = defn.is_class or defn.name() in ('__new__', '__init_subclass__')+                        if isclass:+                            ref_type = mypy.types.TypeType.make_normalized(ref_type)+                        erased = erase_to_bound(arg_type)+                        if not is_subtype_ignoring_tvars(ref_type, erased):+                            note = None+                            if typ.arg_names[i] in ['self', 'cls']:+                                if (self.options.python_version[0] < 3+                                        and is_same_type(erased, arg_type) and not isclass):+                                    msg = ("Invalid type for self, or extra argument type "+                                           "in function annotation")+                                    note = '(Hint: typically annotations omit the type for self)'+                                else:+                                    msg = ("The erased type of self '{}' "+                                           "is not a supertype of its class '{}'"+                                           ).format(erased, ref_type)+                            else:+                                msg = ("Self argument missing for a non-static method "+                                       "(or an invalid type for self)")+                            self.fail(msg, defn)+                            if note:+                                self.note(note, defn)+                        if defn.is_class and isinstance(arg_type, CallableType):+                            arg_type.is_classmethod_class = True+                    elif isinstance(arg_type, TypeVarType):+                        # Refuse covariant parameter type variables+                        # TODO: check recursively for inner type variables+                        if (+                            arg_type.variance == COVARIANT and+                            defn.name() not in ('__init__', '__new__')+                        ):+                            ctx = arg_type  # type: Context+                            if ctx.line < 0:+                                ctx = typ+                            self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, ctx)+                    if typ.arg_kinds[i] == nodes.ARG_STAR:+                        # builtins.tuple[T] is typing.Tuple[T, ...]+                        arg_type = self.named_generic_type('builtins.tuple',+                                                           [arg_type])+                    elif typ.arg_kinds[i] == nodes.ARG_STAR2:+                        arg_type = self.named_generic_type('builtins.dict',+                                                           [self.str_type(),+                                                            arg_type])+                    item.arguments[i].variable.type = arg_type++                # Type check initialization expressions.+                for arg in item.arguments:+                    if arg.initializer is not None:+                        name = arg.variable.name()+                        msg = 'Incompatible default for '+                        if name.startswith('__tuple_arg_'):+                            msg += "tuple argument {}".format(name[12:])+                        else:+                            msg += 'argument "{}"'.format(name)+                        self.check_simple_assignment(arg.variable.type, arg.initializer,+                            context=arg, msg=msg, lvalue_name='argument', rvalue_name='default')++            # Type check body in a new scope.+            with self.binder.top_frame_context():+                with self.scope.push_function(defn):+                    self.accept(item.body)+                unreachable = self.binder.is_unreachable()++            if (self.options.warn_no_return and not unreachable):+                if (defn.is_generator or+                        is_named_instance(self.return_types[-1], 'typing.AwaitableGenerator')):+                    return_type = self.get_generator_return_type(self.return_types[-1],+                                                                 defn.is_coroutine)+                elif defn.is_coroutine:+                    return_type = self.get_coroutine_return_type(self.return_types[-1])+                else:+                    return_type = self.return_types[-1]++                if (not isinstance(return_type, (NoneTyp, AnyType))+                        and not self.is_trivial_body(defn.body)):+                    # Control flow fell off the end of a function that was+                    # declared to return a non-None type and is not+                    # entirely pass/Ellipsis.+                    if isinstance(return_type, UninhabitedType):+                        # This is a NoReturn function+                        self.msg.note(messages.INVALID_IMPLICIT_RETURN, defn)+                    else:+                        self.msg.fail(messages.MISSING_RETURN_STATEMENT, defn)++            self.return_types.pop()++            self.binder = old_binder++    def is_forward_op_method(self, method_name: str) -> bool:+        if self.options.python_version[0] == 2 and method_name == '__div__':+            return True+        else:+            return method_name in nodes.reverse_op_methods++    def is_reverse_op_method(self, method_name: str) -> bool:+        if self.options.python_version[0] == 2 and method_name == '__rdiv__':+            return True+        else:+            return method_name in nodes.reverse_op_method_set++    def check_for_missing_annotations(self, fdef: FuncItem) -> None:+        # Check for functions with unspecified/not fully specified types.+        def is_unannotated_any(t: Type) -> bool:+            return isinstance(t, AnyType) and t.type_of_any == TypeOfAny.unannotated++        has_explicit_annotation = (isinstance(fdef.type, CallableType)+                                   and any(not is_unannotated_any(t)+                                           for t in fdef.type.arg_types + [fdef.type.ret_type]))++        show_untyped = not self.is_typeshed_stub or self.options.warn_incomplete_stub+        check_incomplete_defs = self.options.disallow_incomplete_defs and has_explicit_annotation+        if show_untyped and (self.options.disallow_untyped_defs or check_incomplete_defs):+            if fdef.type is None and self.options.disallow_untyped_defs:+                self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef)+            elif isinstance(fdef.type, CallableType):+                ret_type = fdef.type.ret_type+                if is_unannotated_any(ret_type):+                    self.fail(messages.RETURN_TYPE_EXPECTED, fdef)+                elif fdef.is_generator:+                    if is_unannotated_any(self.get_generator_return_type(ret_type,+                                                                        fdef.is_coroutine)):+                        self.fail(messages.RETURN_TYPE_EXPECTED, fdef)+                elif fdef.is_coroutine and isinstance(ret_type, Instance):+                    if is_unannotated_any(self.get_coroutine_return_type(ret_type)):+                        self.fail(messages.RETURN_TYPE_EXPECTED, fdef)+                if any(is_unannotated_any(t) for t in fdef.type.arg_types):+                    self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef)++    def is_trivial_body(self, block: Block) -> bool:+        body = block.body++        # Skip a docstring+        if (body and isinstance(body[0], ExpressionStmt) and+                isinstance(body[0].expr, (StrExpr, UnicodeExpr))):+            body = block.body[1:]++        if len(body) == 0:+            # There's only a docstring (or no body at all).+            return True+        elif len(body) > 1:+            return False+        stmt = body[0]+        return (isinstance(stmt, PassStmt) or+                (isinstance(stmt, ExpressionStmt) and+                 isinstance(stmt.expr, EllipsisExpr)))++    def check_reverse_op_method(self, defn: FuncItem,+                                reverse_type: CallableType, reverse_name: str,+                                context: Context) -> None:+        """Check a reverse operator method such as __radd__."""+        # Decides whether it's worth calling check_overlapping_op_methods().++        # This used to check for some very obscure scenario.  It now+        # just decides whether it's worth calling+        # check_overlapping_op_methods().++        assert defn.info++        # First check for a valid signature+        method_type = CallableType([AnyType(TypeOfAny.special_form),+                                    AnyType(TypeOfAny.special_form)],+                                   [nodes.ARG_POS, nodes.ARG_POS],+                                   [None, None],+                                   AnyType(TypeOfAny.special_form),+                                   self.named_type('builtins.function'))+        if not is_subtype(reverse_type, method_type):+            self.msg.invalid_signature(reverse_type, context)+            return++        if reverse_name in ('__eq__', '__ne__'):+            # These are defined for all objects => can't cause trouble.+            return++        # With 'Any' or 'object' return type we are happy, since any possible+        # return value is valid.+        ret_type = reverse_type.ret_type+        if isinstance(ret_type, AnyType):+            return+        if isinstance(ret_type, Instance):+            if ret_type.type.fullname() == 'builtins.object':+                return+        if reverse_type.arg_kinds[0] == ARG_STAR:+            reverse_type = reverse_type.copy_modified(arg_types=[reverse_type.arg_types[0]] * 2,+                                                      arg_kinds=[ARG_POS] * 2,+                                                      arg_names=[reverse_type.arg_names[0], "_"])+        assert len(reverse_type.arg_types) >= 2++        if self.options.python_version[0] == 2 and reverse_name == '__rdiv__':+            forward_name = '__div__'+        else:+            forward_name = nodes.normal_from_reverse_op[reverse_name]+        forward_inst = reverse_type.arg_types[1]+        if isinstance(forward_inst, TypeVarType):+            forward_inst = forward_inst.upper_bound+        if isinstance(forward_inst, (FunctionLike, TupleType, TypedDictType)):+            forward_inst = forward_inst.fallback+        if isinstance(forward_inst, TypeType):+            item = forward_inst.item+            if isinstance(item, Instance):+                opt_meta = item.type.metaclass_type+                if opt_meta is not None:+                    forward_inst = opt_meta+        if not (isinstance(forward_inst, (Instance, UnionType))+                and forward_inst.has_readable_member(forward_name)):+            return+        forward_base = reverse_type.arg_types[1]+        forward_type = self.expr_checker.analyze_external_member_access(forward_name, forward_base,+                                                                        context=defn)+        self.check_overlapping_op_methods(reverse_type, reverse_name, defn.info,+                                          forward_type, forward_name, forward_base,+                                          context=defn)++    def check_overlapping_op_methods(self,+                                     reverse_type: CallableType,+                                     reverse_name: str,+                                     reverse_class: TypeInfo,+                                     forward_type: Type,+                                     forward_name: str,+                                     forward_base: Type,+                                     context: Context) -> None:+        """Check for overlapping method and reverse method signatures.++        This function assumes that:++        -   The reverse method has valid argument count and kinds.+        -   If the reverse operator method accepts some argument of type+            X, the forward operator method also belong to class X.++            For example, if we have the reverse operator `A.__radd__(B)`, then the+            corresponding forward operator must have the type `B.__add__(...)`.+        """++        # Note: Suppose we have two operator methods "A.__rOP__(B) -> R1" and+        # "B.__OP__(C) -> R2". We check if these two methods are unsafely overlapping+        # by using the following algorithm:+        #+        # 1. Rewrite "B.__OP__(C) -> R1"  to "temp1(B, C) -> R1"+        #+        # 2. Rewrite "A.__rOP__(B) -> R2" to "temp2(B, A) -> R2"+        #+        # 3. Treat temp1 and temp2 as if they were both variants in the same+        #    overloaded function. (This mirrors how the Python runtime calls+        #    operator methods: we first try __OP__, then __rOP__.)+        #+        #    If the first signature is unsafely overlapping with the second,+        #    report an error.+        #+        # 4. However, if temp1 shadows temp2 (e.g. the __rOP__ method can never+        #    be called), do NOT report an error.+        #+        #    This behavior deviates from how we handle overloads -- many of the+        #    modules in typeshed seem to define __OP__ methods that shadow the+        #    corresponding __rOP__ method.+        #+        # Note: we do not attempt to handle unsafe overlaps related to multiple+        # inheritance. (This is consistent with how we handle overloads: we also+        # do not try checking unsafe overlaps due to multiple inheritance there.)++        for forward_item in union_items(forward_type):+            if isinstance(forward_item, CallableType):+                if self.is_unsafe_overlapping_op(forward_item, forward_base, reverse_type):+                    self.msg.operator_method_signatures_overlap(+                        reverse_class, reverse_name,+                        forward_base, forward_name, context)+            elif isinstance(forward_item, Overloaded):+                for item in forward_item.items():+                    if self.is_unsafe_overlapping_op(item, forward_base, reverse_type):+                        self.msg.operator_method_signatures_overlap(+                            reverse_class, reverse_name,+                            forward_base, forward_name,+                            context)+            elif not isinstance(forward_item, AnyType):+                self.msg.forward_operator_not_callable(forward_name, context)++    def is_unsafe_overlapping_op(self,+                                 forward_item: CallableType,+                                 forward_base: Type,+                                 reverse_type: CallableType) -> bool:+        # TODO: check argument kinds?+        if len(forward_item.arg_types) < 1:+            # Not a valid operator method -- can't succeed anyway.+            return False++        # Erase the type if necessary to make sure we don't have a single+        # TypeVar in forward_tweaked. (Having a function signature containing+        # just a single TypeVar can lead to unpredictable behavior.)+        forward_base_erased = forward_base+        if isinstance(forward_base, TypeVarType):+            forward_base_erased = erase_to_bound(forward_base)++        # Construct normalized function signatures corresponding to the+        # operator methods. The first argument is the left operand and the+        # second operand is the right argument -- we switch the order of+        # the arguments of the reverse method.++        forward_tweaked = forward_item.copy_modified(+            arg_types=[forward_base_erased, forward_item.arg_types[0]],+            arg_kinds=[nodes.ARG_POS] * 2,+            arg_names=[None] * 2,+        )+        reverse_tweaked = reverse_type.copy_modified(+            arg_types=[reverse_type.arg_types[1], reverse_type.arg_types[0]],+            arg_kinds=[nodes.ARG_POS] * 2,+            arg_names=[None] * 2,+        )++        reverse_base_erased = reverse_type.arg_types[0]+        if isinstance(reverse_base_erased, TypeVarType):+            reverse_base_erased = erase_to_bound(reverse_base_erased)++        if is_same_type(reverse_base_erased, forward_base_erased):+            return False+        elif is_subtype(reverse_base_erased, forward_base_erased):+            first = reverse_tweaked+            second = forward_tweaked+        else:+            first = forward_tweaked+            second = reverse_tweaked++        return is_unsafe_overlapping_overload_signatures(first, second)++    def check_inplace_operator_method(self, defn: FuncBase) -> None:+        """Check an inplace operator method such as __iadd__.++        They cannot arbitrarily overlap with __add__.+        """+        method = defn.name()+        if method not in nodes.inplace_operator_methods:+            return+        typ = bind_self(self.function_type(defn))+        cls = defn.info+        other_method = '__' + method[3:]+        if cls.has_readable_member(other_method):+            instance = fill_typevars(cls)+            typ2 = self.expr_checker.analyze_external_member_access(+                other_method, instance, defn)+            fail = False+            if isinstance(typ2, FunctionLike):+                if not is_more_general_arg_prefix(typ, typ2):+                    fail = True+            else:+                # TODO overloads+                fail = True+            if fail:+                self.msg.signatures_incompatible(method, other_method, defn)++    def check_getattr_method(self, typ: Type, context: Context, name: str) -> None:+        if len(self.scope.stack) == 1:+            # module scope+            if name == '__getattribute__':+                self.msg.fail('__getattribute__ is not valid at the module level', context)+                return+            # __getattr__ is fine at the module level as of Python 3.7 (PEP 562). We could+            # show an error for Python < 3.7, but that would be annoying in code that supports+            # both 3.7 and older versions.+            method_type = CallableType([self.named_type('builtins.str')],+                                       [nodes.ARG_POS],+                                       [None],+                                       AnyType(TypeOfAny.special_form),+                                       self.named_type('builtins.function'))+        elif self.scope.active_class():+            method_type = CallableType([AnyType(TypeOfAny.special_form),+                                        self.named_type('builtins.str')],+                                       [nodes.ARG_POS, nodes.ARG_POS],+                                       [None, None],+                                       AnyType(TypeOfAny.special_form),+                                       self.named_type('builtins.function'))+        else:+            return+        if not is_subtype(typ, method_type):+            self.msg.invalid_signature_for_special_method(typ, context, name)++    def check_setattr_method(self, typ: Type, context: Context) -> None:+        if not self.scope.active_class():+            return+        method_type = CallableType([AnyType(TypeOfAny.special_form),+                                    self.named_type('builtins.str'),+                                    AnyType(TypeOfAny.special_form)],+                                   [nodes.ARG_POS, nodes.ARG_POS, nodes.ARG_POS],+                                   [None, None, None],+                                   NoneTyp(),+                                   self.named_type('builtins.function'))+        if not is_subtype(typ, method_type):+            self.msg.invalid_signature_for_special_method(typ, context, '__setattr__')++    def expand_typevars(self, defn: FuncItem,+                        typ: CallableType) -> List[Tuple[FuncItem, CallableType]]:+        # TODO use generator+        subst = []  # type: List[List[Tuple[TypeVarId, Type]]]+        tvars = typ.variables or []+        tvars = tvars[:]+        if defn.info:+            # Class type variables+            tvars += defn.info.defn.type_vars or []+        for tvar in tvars:+            if tvar.values:+                subst.append([(tvar.id, value)+                              for value in tvar.values])+        if subst:+            result = []  # type: List[Tuple[FuncItem, CallableType]]+            for substitutions in itertools.product(*subst):+                mapping = dict(substitutions)+                expanded = cast(CallableType, expand_type(typ, mapping))+                result.append((expand_func(defn, mapping), expanded))+            return result+        else:+            return [(defn, typ)]++    def check_method_override(self, defn: Union[FuncBase, Decorator]) -> None:+        """Check if function definition is compatible with base classes."""+        # Check against definitions in base classes.+        for base in defn.info.mro[1:]:+            self.check_method_or_accessor_override_for_base(defn, base)++    def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decorator],+                                                   base: TypeInfo) -> None:+        """Check if method definition is compatible with a base class."""+        if base:+            name = defn.name()+            if name not in ('__init__', '__new__', '__init_subclass__'):+                # Check method override+                # (__init__, __new__, __init_subclass__ are special).+                self.check_method_override_for_base_with_name(defn, name, base)+                if name in nodes.inplace_operator_methods:+                    # Figure out the name of the corresponding operator method.+                    method = '__' + name[3:]+                    # An inplace operator method such as __iadd__ might not be+                    # always introduced safely if a base class defined __add__.+                    # TODO can't come up with an example where this is+                    #      necessary; now it's "just in case"+                    self.check_method_override_for_base_with_name(defn, method,+                                                                  base)++    def check_method_override_for_base_with_name(+            self, defn: Union[FuncBase, Decorator], name: str, base: TypeInfo) -> None:+        base_attr = base.names.get(name)+        if base_attr:+            # The name of the method is defined in the base class.++            # Point errors at the 'def' line (important for backward compatibility+            # of type ignores).+            if not isinstance(defn, Decorator):+                context = defn+            else:+                context = defn.func+            # Construct the type of the overriding method.+            if isinstance(defn, FuncBase):+                typ = self.function_type(defn)  # type: Type+                override_class_or_static = defn.is_class or defn.is_static+            else:+                assert defn.var.is_ready+                assert defn.var.type is not None+                typ = defn.var.type+                override_class_or_static = defn.func.is_class or defn.func.is_static+            if isinstance(typ, FunctionLike) and not is_static(context):+                typ = bind_self(typ, self.scope.active_self_type())+            # Map the overridden method type to subtype context so that+            # it can be checked for compatibility.+            original_type = base_attr.type+            original_node = base_attr.node+            if original_type is None:+                if isinstance(original_node, FuncBase):+                    original_type = self.function_type(original_node)+                elif isinstance(original_node, Decorator):+                    original_type = self.function_type(original_node.func)+                else:+                    assert False, str(base_attr.node)+            if isinstance(original_node, FuncBase):+                original_class_or_static = original_node.is_class or original_node.is_static+            elif isinstance(original_node, Decorator):+                fdef = original_node.func+                original_class_or_static = fdef.is_class or fdef.is_static+            else:+                original_class_or_static = False  # a variable can't be class or static+            if isinstance(original_type, AnyType) or isinstance(typ, AnyType):+                pass+            elif isinstance(original_type, FunctionLike) and isinstance(typ, FunctionLike):+                # mypyc hack to workaround mypy misunderstanding multiple inheritance (#3603)+                base_attr_node = base_attr.node  # type: Any+                if (isinstance(base_attr_node, (FuncBase, Decorator))+                        and not is_static(base_attr_node)):+                    bound = bind_self(original_type, self.scope.active_self_type())+                else:+                    bound = original_type+                original = map_type_from_supertype(bound, defn.info, base)+                # Check that the types are compatible.+                # TODO overloaded signatures+                self.check_override(typ,+                                    cast(FunctionLike, original),+                                    defn.name(),+                                    name,+                                    base.name(),+                                    original_class_or_static,+                                    override_class_or_static,+                                    context)+            elif is_equivalent(original_type, typ):+                # Assume invariance for a non-callable attribute here. Note+                # that this doesn't affect read-only properties which can have+                # covariant overrides.+                #+                # TODO: Allow covariance for read-only attributes?+                pass+            else:+                self.msg.signature_incompatible_with_supertype(+                    defn.name(), name, base.name(), context)++    def check_override(self, override: FunctionLike, original: FunctionLike,+                       name: str, name_in_super: str, supertype: str,+                       original_class_or_static: bool,+                       override_class_or_static: bool,+                       node: Context) -> None:+        """Check a method override with given signatures.++        Arguments:+          override:  The signature of the overriding method.+          original:  The signature of the original supertype method.+          name:      The name of the subtype. This and the next argument are+                     only used for generating error messages.+          supertype: The name of the supertype.+        """+        # Use boolean variable to clarify code.+        fail = False+        if not is_subtype(override, original, ignore_pos_arg_names=True):+            fail = True+        elif (not isinstance(original, Overloaded) and+              isinstance(override, Overloaded) and+              self.is_forward_op_method(name)):+            # Operator method overrides cannot introduce overloading, as+            # this could be unsafe with reverse operator methods.+            fail = True++        if isinstance(original, FunctionLike) and isinstance(override, FunctionLike):+            if original_class_or_static and not override_class_or_static:+                fail = True++        if fail:+            emitted_msg = False+            if (isinstance(override, CallableType) and+                    isinstance(original, CallableType) and+                    len(override.arg_types) == len(original.arg_types) and+                    override.min_args == original.min_args):+                # Give more detailed messages for the common case of both+                # signatures having the same number of arguments and no+                # overloads.++                # override might have its own generic function type+                # variables. If an argument or return type of override+                # does not have the correct subtyping relationship+                # with the original type even after these variables+                # are erased, then it is definitely an incompatibility.++                override_ids = override.type_var_ids()++                def erase_override(t: Type) -> Type:+                    return erase_typevars(t, ids_to_erase=override_ids)++                for i in range(len(override.arg_types)):+                    if not is_subtype(original.arg_types[i],+                                      erase_override(override.arg_types[i])):+                        self.msg.argument_incompatible_with_supertype(+                            i + 1, name, name_in_super, supertype, node)+                        emitted_msg = True++                if not is_subtype(erase_override(override.ret_type),+                                  original.ret_type):+                    self.msg.return_type_incompatible_with_supertype(+                        name, name_in_super, supertype, node)+                    emitted_msg = True+            elif isinstance(override, Overloaded) and isinstance(original, Overloaded):+                # Give a more detailed message in the case where the user is trying to+                # override an overload, and the subclass's overload is plausible, except+                # that the order of the variants are wrong.+                #+                # For example, if the parent defines the overload f(int) -> int and f(str) -> str+                # (in that order), and if the child swaps the two and does f(str) -> str and+                # f(int) -> int+                order = []+                for child_variant in override.items():+                    for i, parent_variant in enumerate(original.items()):+                        if is_subtype(child_variant, parent_variant):+                            order.append(i)+                            break++                if len(order) == len(original.items()) and order != sorted(order):+                    self.msg.overload_signature_incompatible_with_supertype(+                        name, name_in_super, supertype, override, node)+                    emitted_msg = True++            if not emitted_msg:+                # Fall back to generic incompatibility message.+                self.msg.signature_incompatible_with_supertype(+                    name, name_in_super, supertype, node)++    def visit_class_def(self, defn: ClassDef) -> None:+        """Type check a class definition."""+        typ = defn.info+        if typ.is_protocol and typ.defn.type_vars:+            self.check_protocol_variance(defn)+        with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True):+            old_binder = self.binder+            self.binder = ConditionalTypeBinder()+            with self.binder.top_frame_context():+                with self.scope.push_class(defn.info):+                    self.accept(defn.defs)+            self.binder = old_binder+            if not defn.has_incompatible_baseclass:+                # Otherwise we've already found errors; more errors are not useful+                self.check_multiple_inheritance(typ)++            if defn.decorators:+                sig = type_object_type(defn.info, self.named_type)+                # Decorators are applied in reverse order.+                for decorator in reversed(defn.decorators):+                    if (isinstance(decorator, CallExpr)+                            and isinstance(decorator.analyzed, PromoteExpr)):+                        # _promote is a special type checking related construct.+                        continue++                    dec = self.expr_checker.accept(decorator)+                    temp = self.temp_node(sig)+                    fullname = None+                    if isinstance(decorator, RefExpr):+                        fullname = decorator.fullname++                    # TODO: Figure out how to have clearer error messages.+                    # (e.g. "class decorator must be a function that accepts a type."+                    sig, _ = self.expr_checker.check_call(dec, [temp],+                                                          [nodes.ARG_POS], defn,+                                                          callable_name=fullname)+                # TODO: Apply the sig to the actual TypeInfo so we can handle decorators+                # that completely swap out the type.  (e.g. Callable[[Type[A]], Type[B]])++    def check_protocol_variance(self, defn: ClassDef) -> None:+        """Check that protocol definition is compatible with declared+        variances of type variables.++        Note that we also prohibit declaring protocol classes as invariant+        if they are actually covariant/contravariant, since this may break+        transitivity of subtyping, see PEP 544.+        """+        info = defn.info+        object_type = Instance(info.mro[-1], [])+        tvars = info.defn.type_vars+        for i, tvar in enumerate(tvars):+            up_args = [object_type if i == j else AnyType(TypeOfAny.special_form)+                       for j, _ in enumerate(tvars)]  # type: List[Type]+            down_args = [UninhabitedType() if i == j else AnyType(TypeOfAny.special_form)+                         for j, _ in enumerate(tvars)]  # type: List[Type]+            up, down = Instance(info, up_args), Instance(info, down_args)+            # TODO: add advanced variance checks for recursive protocols+            if is_subtype(down, up, ignore_declared_variance=True):+                expected = COVARIANT+            elif is_subtype(up, down, ignore_declared_variance=True):+                expected = CONTRAVARIANT+            else:+                expected = INVARIANT+            if expected != tvar.variance:+                self.msg.bad_proto_variance(tvar.variance, tvar.name, expected, defn)++    def check_multiple_inheritance(self, typ: TypeInfo) -> None:+        """Check for multiple inheritance related errors."""+        if len(typ.bases) <= 1:+            # No multiple inheritance.+            return+        # Verify that inherited attributes are compatible.+        mro = typ.mro[1:]+        for i, base in enumerate(mro):+            for name in base.names:+                for base2 in mro[i + 1:]:+                    # We only need to check compatibility of attributes from classes not+                    # in a subclass relationship. For subclasses, normal (single inheritance)+                    # checks suffice (these are implemented elsewhere).+                    if name in base2.names and base2 not in base.mro:+                        self.check_compatibility(name, base, base2, typ)++    def check_compatibility(self, name: str, base1: TypeInfo,+                            base2: TypeInfo, ctx: Context) -> None:+        """Check if attribute name in base1 is compatible with base2 in multiple inheritance.++        Assume base1 comes before base2 in the MRO, and that base1 and base2 don't have+        a direct subclass relationship (i.e., the compatibility requirement only derives from+        multiple inheritance).+        """+        if name == '__init__':+            # __init__ can be incompatible -- it's a special case.+            return+        first = base1[name]+        second = base2[name]+        first_type = first.type+        if first_type is None and isinstance(first.node, FuncBase):+            first_type = self.function_type(first.node)+        second_type = second.type+        if second_type is None and isinstance(second.node, FuncBase):+            second_type = self.function_type(second.node)+        # TODO: What if some classes are generic?+        if (isinstance(first_type, FunctionLike) and+                isinstance(second_type, FunctionLike)):+            # Method override+            first_sig = bind_self(first_type)+            second_sig = bind_self(second_type)+            ok = is_subtype(first_sig, second_sig, ignore_pos_arg_names=True)+        elif first_type and second_type:+            ok = is_equivalent(first_type, second_type)+        else:+            if first_type is None:+                self.msg.cannot_determine_type_in_base(name, base1.name(), ctx)+            if second_type is None:+                self.msg.cannot_determine_type_in_base(name, base2.name(), ctx)+            ok = True+        # __slots__ is special and the type can vary across class hierarchy.+        if name == '__slots__':+            ok = True+        if not ok:+            self.msg.base_class_definitions_incompatible(name, base1, base2,+                                                         ctx)++    def visit_import_from(self, node: ImportFrom) -> None:+        self.check_import(node)++    def visit_import_all(self, node: ImportAll) -> None:+        self.check_import(node)++    def visit_import(self, s: Import) -> None:+        pass++    def check_import(self, node: ImportBase) -> None:+        for assign in node.assignments:+            lvalue = assign.lvalues[0]+            lvalue_type, _, __ = self.check_lvalue(lvalue)+            if lvalue_type is None:+                # TODO: This is broken.+                lvalue_type = AnyType(TypeOfAny.special_form)+            message = '{} "{}"'.format(messages.INCOMPATIBLE_IMPORT_OF,+                                       cast(NameExpr, assign.rvalue).name)+            self.check_simple_assignment(lvalue_type, assign.rvalue, node,+                                         msg=message, lvalue_name='local name',+                                         rvalue_name='imported name')++    #+    # Statements+    #++    def visit_block(self, b: Block) -> None:+        if b.is_unreachable:+            self.binder.unreachable()+            return+        for s in b.body:+            if self.binder.is_unreachable():+                break+            self.accept(s)++    def visit_assignment_stmt(self, s: AssignmentStmt) -> None:+        """Type check an assignment statement.++        Handle all kinds of assignment statements (simple, indexed, multiple).+        """+        self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None, s.new_syntax)++        if (s.type is not None and+                self.options.disallow_any_unimported and+                has_any_from_unimported_type(s.type)):+            if isinstance(s.lvalues[-1], TupleExpr):+                # This is a multiple assignment. Instead of figuring out which type is problematic,+                # give a generic error message.+                self.msg.unimported_type_becomes_any("A type on this line",+                                                     AnyType(TypeOfAny.special_form), s)+            else:+                self.msg.unimported_type_becomes_any("Type of variable", s.type, s)+        check_for_explicit_any(s.type, self.options, self.is_typeshed_stub, self.msg, context=s)++        if len(s.lvalues) > 1:+            # Chained assignment (e.g. x = y = ...).+            # Make sure that rvalue type will not be reinferred.+            if s.rvalue not in self.type_map:+                self.expr_checker.accept(s.rvalue)+            rvalue = self.temp_node(self.type_map[s.rvalue], s)+            for lv in s.lvalues[:-1]:+                self.check_assignment(lv, rvalue, s.type is None)++    def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type: bool = True,+                         new_syntax: bool = False) -> None:+        """Type check a single assignment: lvalue = rvalue."""+        if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr):+            self.check_assignment_to_multiple_lvalues(lvalue.items, rvalue, lvalue,+                                                      infer_lvalue_type)+        else:+            lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue)++            # If we're assigning to __getattr__ or similar methods, check that the signature is+            # valid.+            if isinstance(lvalue, NameExpr) and lvalue.node:+                name = lvalue.node.name()+                if name in ('__setattr__', '__getattribute__', '__getattr__'):+                    # If an explicit type is given, use that.+                    if lvalue_type:+                        signature = lvalue_type+                    else:+                        signature = self.expr_checker.accept(rvalue)+                    if signature:+                        if name == '__setattr__':+                            self.check_setattr_method(signature, lvalue)+                        else:+                            self.check_getattr_method(signature, lvalue, name)++            if isinstance(lvalue, RefExpr):+                if self.check_compatibility_all_supers(lvalue, lvalue_type, rvalue):+                    # We hit an error on this line; don't check for any others+                    return++            if lvalue_type:+                if isinstance(lvalue_type, PartialType) and lvalue_type.type is None:+                    # Try to infer a proper type for a variable with a partial None type.+                    rvalue_type = self.expr_checker.accept(rvalue)+                    if isinstance(rvalue_type, NoneTyp):+                        # This doesn't actually provide any additional information -- multiple+                        # None initializers preserve the partial None type.+                        return++                    if is_valid_inferred_type(rvalue_type):+                        var = lvalue_type.var+                        partial_types = self.find_partial_types(var)+                        if partial_types is not None:+                            if not self.current_node_deferred:+                                inferred_type = UnionType.make_simplified_union(+                                    [rvalue_type, NoneTyp()])+                                self.set_inferred_type(var, lvalue, inferred_type)+                            else:+                                var.type = None+                            del partial_types[var]+                            lvalue_type = var.type+                    else:+                        # Try to infer a partial type. No need to check the return value, as+                        # an error will be reported elsewhere.+                        self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type)+                elif (is_literal_none(rvalue) and+                        isinstance(lvalue, NameExpr) and+                        isinstance(lvalue.node, Var) and+                        lvalue.node.is_initialized_in_class and+                        not new_syntax):+                    # Allow None's to be assigned to class variables with non-Optional types.+                    rvalue_type = lvalue_type+                elif (isinstance(lvalue, MemberExpr) and+                        lvalue.kind is None):  # Ignore member access to modules+                    instance_type = self.expr_checker.accept(lvalue.expr)+                    rvalue_type, infer_lvalue_type = self.check_member_assignment(+                        instance_type, lvalue_type, rvalue, lvalue)+                else:+                    rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue)++                # Special case: only non-abstract non-protocol classes can be assigned to+                # variables with explicit type Type[A], where A is protocol or abstract.+                if (isinstance(rvalue_type, CallableType) and rvalue_type.is_type_obj() and+                        (rvalue_type.type_object().is_abstract or+                         rvalue_type.type_object().is_protocol) and+                        isinstance(lvalue_type, TypeType) and+                        isinstance(lvalue_type.item, Instance) and+                        (lvalue_type.item.type.is_abstract or+                         lvalue_type.item.type.is_protocol)):+                    self.msg.concrete_only_assign(lvalue_type, rvalue)+                    return+                if rvalue_type and infer_lvalue_type and not isinstance(lvalue_type, PartialType):+                    self.binder.assign_type(lvalue, rvalue_type, lvalue_type, False)++            elif index_lvalue:+                self.check_indexed_assignment(index_lvalue, rvalue, lvalue)++            if inferred:+                self.infer_variable_type(inferred, lvalue, self.expr_checker.accept(rvalue),+                                         rvalue)++    def check_compatibility_all_supers(self, lvalue: RefExpr, lvalue_type: Optional[Type],+                                       rvalue: Expression) -> bool:+        lvalue_node = lvalue.node+        # Check if we are a class variable with at least one base class+        if (isinstance(lvalue_node, Var) and+                lvalue.kind in (MDEF, None) and  # None for Vars defined via self+                len(lvalue_node.info.bases) > 0):++            for base in lvalue_node.info.mro[1:]:+                tnode = base.names.get(lvalue_node.name())+                if tnode is not None:+                    if not self.check_compatibility_classvar_super(lvalue_node,+                                                                   base,+                                                                   tnode.node):+                        # Show only one error per variable+                        break++            for base in lvalue_node.info.mro[1:]:+                # Only check __slots__ against the 'object'+                # If a base class defines a Tuple of 3 elements, a child of+                # this class should not be allowed to define it as a Tuple of+                # anything other than 3 elements. The exception to this rule+                # is __slots__, where it is allowed for any child class to+                # redefine it.+                if lvalue_node.name() == "__slots__" and base.fullname() != "builtins.object":+                    continue++                base_type, base_node = self.lvalue_type_from_base(lvalue_node, base)++                if base_type:+                    assert base_node is not None+                    if not self.check_compatibility_super(lvalue,+                                                          lvalue_type,+                                                          rvalue,+                                                          base,+                                                          base_type,+                                                          base_node):+                        # Only show one error per variable; even if other+                        # base classes are also incompatible+                        return True+                    break+        return False++    def check_compatibility_super(self, lvalue: RefExpr, lvalue_type: Optional[Type],+                                  rvalue: Expression, base: TypeInfo, base_type: Type,+                                  base_node: Node) -> bool:+        lvalue_node = lvalue.node+        assert isinstance(lvalue_node, Var)++        # Do not check whether the rvalue is compatible if the+        # lvalue had a type defined; this is handled by other+        # parts, and all we have to worry about in that case is+        # that lvalue is compatible with the base class.+        compare_node = None+        if lvalue_type:+            compare_type = lvalue_type+            compare_node = lvalue.node+        else:+            compare_type = self.expr_checker.accept(rvalue, base_type)+            if isinstance(rvalue, NameExpr):+                compare_node = rvalue.node+                if isinstance(compare_node, Decorator):+                    compare_node = compare_node.func++        if compare_type:+            if (isinstance(base_type, CallableType) and+                    isinstance(compare_type, CallableType)):+                base_static = is_node_static(base_node)+                compare_static = is_node_static(compare_node)++                # In case compare_static is unknown, also check+                # if 'definition' is set. The most common case for+                # this is with TempNode(), where we lose all+                # information about the real rvalue node (but only get+                # the rvalue type)+                if compare_static is None and compare_type.definition:+                    compare_static = is_node_static(compare_type.definition)++                # Compare against False, as is_node_static can return None+                if base_static is False and compare_static is False:+                    # Class-level function objects and classmethods become bound+                    # methods: the former to the instance, the latter to the+                    # class+                    base_type = bind_self(base_type, self.scope.active_self_type())+                    compare_type = bind_self(compare_type, self.scope.active_self_type())++                # If we are a static method, ensure to also tell the+                # lvalue it now contains a static method+                if base_static and compare_static:+                    lvalue_node.is_staticmethod = True++            return self.check_subtype(compare_type, base_type, lvalue,+                                      messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT,+                                      'expression has type',+                                      'base class "%s" defined the type as' % base.name())+        return True++    def lvalue_type_from_base(self, expr_node: Var,+                              base: TypeInfo) -> Tuple[Optional[Type], Optional[Node]]:+        """For a NameExpr that is part of a class, walk all base classes and try+        to find the first class that defines a Type for the same name."""+        expr_name = expr_node.name()+        base_var = base.names.get(expr_name)++        if base_var:+            base_node = base_var.node+            base_type = base_var.type+            if isinstance(base_node, Decorator):+                base_node = base_node.func+                base_type = base_node.type++            if base_type:+                if not has_no_typevars(base_type):+                    self_type = self.scope.active_self_type()+                    assert self_type is not None, "Internal error: base lookup outside class"+                    if isinstance(self_type, TupleType):+                        instance = self_type.fallback+                    else:+                        instance = self_type+                    itype = map_instance_to_supertype(instance, base)+                    base_type = expand_type_by_instance(base_type, itype)++                if isinstance(base_type, CallableType) and isinstance(base_node, FuncDef):+                    # If we are a property, return the Type of the return+                    # value, not the Callable+                    if base_node.is_property:+                        base_type = base_type.ret_type++                return base_type, base_node++        return None, None++    def check_compatibility_classvar_super(self, node: Var,+                                           base: TypeInfo, base_node: Optional[Node]) -> bool:+        if not isinstance(base_node, Var):+            return True+        if node.is_classvar and not base_node.is_classvar:+            self.fail('Cannot override instance variable '+                      '(previously declared on base class "%s") '+                      'with class variable' % base.name(), node)+            return False+        elif not node.is_classvar and base_node.is_classvar:+            self.fail('Cannot override class variable '+                      '(previously declared on base class "%s") '+                      'with instance variable' % base.name(), node)+            return False+        return True++    def check_assignment_to_multiple_lvalues(self, lvalues: List[Lvalue], rvalue: Expression,+                                             context: Context,+                                             infer_lvalue_type: bool = True) -> None:+        if isinstance(rvalue, TupleExpr) or isinstance(rvalue, ListExpr):+            # Recursively go into Tuple or List expression rhs instead of+            # using the type of rhs, because this allowed more fine grained+            # control in cases like: a, b = [int, str] where rhs would get+            # type List[object]++            rvalues = rvalue.items++            if self.check_rvalue_count_in_assignment(lvalues, len(rvalues), context):+                star_index = next((i for i, lv in enumerate(lvalues) if+                                   isinstance(lv, StarExpr)), len(lvalues))++                left_lvs = lvalues[:star_index]+                star_lv = cast(StarExpr,+                               lvalues[star_index]) if star_index != len(lvalues) else None+                right_lvs = lvalues[star_index + 1:]++                left_rvs, star_rvs, right_rvs = self.split_around_star(+                    rvalues, star_index, len(lvalues))++                lr_pairs = list(zip(left_lvs, left_rvs))+                if star_lv:+                    rv_list = ListExpr(star_rvs)+                    rv_list.set_line(rvalue.get_line())+                    lr_pairs.append((star_lv.expr, rv_list))+                lr_pairs.extend(zip(right_lvs, right_rvs))++                for lv, rv in lr_pairs:+                    self.check_assignment(lv, rv, infer_lvalue_type)+        else:+            self.check_multi_assignment(lvalues, rvalue, context, infer_lvalue_type)++    def check_rvalue_count_in_assignment(self, lvalues: List[Lvalue], rvalue_count: int,+                                         context: Context) -> bool:+        if any(isinstance(lvalue, StarExpr) for lvalue in lvalues):+            if len(lvalues) - 1 > rvalue_count:+                self.msg.wrong_number_values_to_unpack(rvalue_count,+                                                       len(lvalues) - 1, context)+                return False+        elif rvalue_count != len(lvalues):+            self.msg.wrong_number_values_to_unpack(rvalue_count,+                            len(lvalues), context)+            return False+        return True++    def check_multi_assignment(self, lvalues: List[Lvalue],+                               rvalue: Expression,+                               context: Context,+                               infer_lvalue_type: bool = True,+                               rv_type: Optional[Type] = None,+                               undefined_rvalue: bool = False) -> None:+        """Check the assignment of one rvalue to a number of lvalues."""++        # Infer the type of an ordinary rvalue expression.+        # TODO: maybe elsewhere; redundant.+        rvalue_type = rv_type or self.expr_checker.accept(rvalue)++        if isinstance(rvalue_type, UnionType):+            # If this is an Optional type in non-strict Optional code, unwrap it.+            relevant_items = rvalue_type.relevant_items()+            if len(relevant_items) == 1:+                rvalue_type = relevant_items[0]++        if isinstance(rvalue_type, AnyType):+            for lv in lvalues:+                if isinstance(lv, StarExpr):+                    lv = lv.expr+                temp_node = self.temp_node(AnyType(TypeOfAny.from_another_any,+                                                   source_any=rvalue_type), context)+                self.check_assignment(lv, temp_node, infer_lvalue_type)+        elif isinstance(rvalue_type, TupleType):+            self.check_multi_assignment_from_tuple(lvalues, rvalue, rvalue_type,+                                                   context, undefined_rvalue, infer_lvalue_type)+        elif isinstance(rvalue_type, UnionType):+            self.check_multi_assignment_from_union(lvalues, rvalue, rvalue_type, context,+                                                   infer_lvalue_type)+        else:+            self.check_multi_assignment_from_iterable(lvalues, rvalue_type,+                                                      context, infer_lvalue_type)++    def check_multi_assignment_from_union(self, lvalues: List[Expression], rvalue: Expression,+                                          rvalue_type: UnionType, context: Context,+                                          infer_lvalue_type: bool) -> None:+        """Check assignment to multiple lvalue targets when rvalue type is a Union[...].+        For example:++            t: Union[Tuple[int, int], Tuple[str, str]]+            x, y = t+            reveal_type(x)  # Union[int, str]++        The idea in this case is to process the assignment for every item of the union.+        Important note: the types are collected in two places, 'union_types' contains+        inferred types for first assignments, 'assignments' contains the narrowed types+        for binder.+        """+        self.no_partial_types = True+        transposed = tuple([] for _ in+                           self.flatten_lvalues(lvalues))  # type: Tuple[List[Type], ...]+        # Notify binder that we want to defer bindings and instead collect types.+        with self.binder.accumulate_type_assignments() as assignments:+            for item in rvalue_type.items:+                # Type check the assignment separately for each union item and collect+                # the inferred lvalue types for each union item.+                self.check_multi_assignment(lvalues, rvalue, context,+                                            infer_lvalue_type=infer_lvalue_type,+                                            rv_type=item, undefined_rvalue=True)+                for t, lv in zip(transposed, self.flatten_lvalues(lvalues)):+                    t.append(self.type_map.pop(lv, AnyType(TypeOfAny.special_form)))+        union_types = tuple(UnionType.make_simplified_union(col) for col in transposed)+        for expr, items in assignments.items():+            # Bind a union of types collected in 'assignments' to every expression.+            if isinstance(expr, StarExpr):+                expr = expr.expr++            # TODO: See todo in binder.py, ConditionalTypeBinder.assign_type+            # It's unclear why the 'declared_type' param is sometimes 'None'+            clean_items = []  # type: List[Tuple[Type, Type]]+            for type, declared_type in items:+                assert declared_type is not None+                clean_items.append((type, declared_type))++            types, declared_types = zip(*clean_items)+            self.binder.assign_type(expr,+                                    UnionType.make_simplified_union(list(types)),+                                    UnionType.make_simplified_union(list(declared_types)),+                                    False)+        for union, lv in zip(union_types, self.flatten_lvalues(lvalues)):+            # Properly store the inferred types.+            _1, _2, inferred = self.check_lvalue(lv)+            if inferred:+                self.set_inferred_type(inferred, lv, union)+            else:+                self.store_type(lv, union)+        self.no_partial_types = False++    def flatten_lvalues(self, lvalues: List[Expression]) -> List[Expression]:+        res = []  # type: List[Expression]+        for lv in lvalues:+            if isinstance(lv, (TupleExpr, ListExpr)):+                res.extend(self.flatten_lvalues(lv.items))+            if isinstance(lv, StarExpr):+                # Unwrap StarExpr, since it is unwrapped by other helpers.+                lv = lv.expr+            res.append(lv)+        return res++    def check_multi_assignment_from_tuple(self, lvalues: List[Lvalue], rvalue: Expression,+                                          rvalue_type: TupleType, context: Context,+                                          undefined_rvalue: bool,+                                          infer_lvalue_type: bool = True) -> None:+        if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context):+            star_index = next((i for i, lv in enumerate(lvalues)+                               if isinstance(lv, StarExpr)), len(lvalues))++            left_lvs = lvalues[:star_index]+            star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None+            right_lvs = lvalues[star_index + 1:]++            if not undefined_rvalue:+                # Infer rvalue again, now in the correct type context.+                lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type)+                reinferred_rvalue_type = self.expr_checker.accept(rvalue, lvalue_type)++                if isinstance(reinferred_rvalue_type, UnionType):+                    # If this is an Optional type in non-strict Optional code, unwrap it.+                    relevant_items = reinferred_rvalue_type.relevant_items()+                    if len(relevant_items) == 1:+                        reinferred_rvalue_type = relevant_items[0]+                if isinstance(reinferred_rvalue_type, UnionType):+                    self.check_multi_assignment_from_union(lvalues, rvalue,+                                                           reinferred_rvalue_type, context,+                                                           infer_lvalue_type)+                    return+                assert isinstance(reinferred_rvalue_type, TupleType)+                rvalue_type = reinferred_rvalue_type++            left_rv_types, star_rv_types, right_rv_types = self.split_around_star(+                rvalue_type.items, star_index, len(lvalues))++            for lv, rv_type in zip(left_lvs, left_rv_types):+                self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type)+            if star_lv:+                list_expr = ListExpr([self.temp_node(rv_type, context)+                                      for rv_type in star_rv_types])+                list_expr.set_line(context.get_line())+                self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type)+            for lv, rv_type in zip(right_lvs, right_rv_types):+                self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type)++    def lvalue_type_for_inference(self, lvalues: List[Lvalue], rvalue_type: TupleType) -> Type:+        star_index = next((i for i, lv in enumerate(lvalues)+                           if isinstance(lv, StarExpr)), len(lvalues))+        left_lvs = lvalues[:star_index]+        star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None+        right_lvs = lvalues[star_index + 1:]+        left_rv_types, star_rv_types, right_rv_types = self.split_around_star(+            rvalue_type.items, star_index, len(lvalues))++        type_parameters = []  # type: List[Type]++        def append_types_for_inference(lvs: List[Expression], rv_types: List[Type]) -> None:+            for lv, rv_type in zip(lvs, rv_types):+                sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv)+                if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType):+                    type_parameters.append(sub_lvalue_type)+                else:  # index lvalue+                    # TODO Figure out more precise type context, probably+                    #      based on the type signature of the _set method.+                    type_parameters.append(rv_type)++        append_types_for_inference(left_lvs, left_rv_types)++        if star_lv:+            sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr)+            if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType):+                type_parameters.extend([sub_lvalue_type] * len(star_rv_types))+            else:  # index lvalue+                # TODO Figure out more precise type context, probably+                #      based on the type signature of the _set method.+                type_parameters.extend(star_rv_types)++        append_types_for_inference(right_lvs, right_rv_types)++        return TupleType(type_parameters, self.named_type('builtins.tuple'))++    def split_around_star(self, items: List[T], star_index: int,+                          length: int) -> Tuple[List[T], List[T], List[T]]:+        """Splits a list of items in three to match another list of length 'length'+        that contains a starred expression at 'star_index' in the following way:++        star_index = 2, length = 5 (i.e., [a,b,*,c,d]), items = [1,2,3,4,5,6,7]+        returns in: ([1,2], [3,4,5], [6,7])+        """+        nr_right_of_star = length - star_index - 1+        right_index = -nr_right_of_star if nr_right_of_star != 0 else len(items)+        left = items[:star_index]+        star = items[star_index:right_index]+        right = items[right_index:]+        return (left, star, right)++    def type_is_iterable(self, type: Type) -> bool:+        if isinstance(type, CallableType) and type.is_type_obj():+            type = type.fallback+        return (is_subtype(type, self.named_generic_type('typing.Iterable',+                                                         [AnyType(TypeOfAny.special_form)])) and+                isinstance(type, Instance))++    def check_multi_assignment_from_iterable(self, lvalues: List[Lvalue], rvalue_type: Type,+                                             context: Context,+                                             infer_lvalue_type: bool = True) -> None:+        if self.type_is_iterable(rvalue_type):+            item_type = self.iterable_item_type(cast(Instance, rvalue_type))+            for lv in lvalues:+                if isinstance(lv, StarExpr):+                    items_type = self.named_generic_type('builtins.list', [item_type])+                    self.check_assignment(lv.expr, self.temp_node(items_type, context),+                                          infer_lvalue_type)+                else:+                    self.check_assignment(lv, self.temp_node(item_type, context),+                                          infer_lvalue_type)+        else:+            self.msg.type_not_iterable(rvalue_type, context)++    def check_lvalue(self, lvalue: Lvalue) -> Tuple[Optional[Type],+                                                    Optional[IndexExpr],+                                                    Optional[Var]]:+        lvalue_type = None+        index_lvalue = None+        inferred = None++        if self.is_definition(lvalue):+            if isinstance(lvalue, NameExpr):+                inferred = cast(Var, lvalue.node)+                assert isinstance(inferred, Var)+            else:+                assert isinstance(lvalue, MemberExpr)+                self.expr_checker.accept(lvalue.expr)+                inferred = lvalue.def_var+        elif isinstance(lvalue, IndexExpr):+            index_lvalue = lvalue+        elif isinstance(lvalue, MemberExpr):+            lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue,+                                                                 True)+            self.store_type(lvalue, lvalue_type)+        elif isinstance(lvalue, NameExpr):+            lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True)+            self.store_type(lvalue, lvalue_type)+        elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr):+            types = [self.check_lvalue(sub_expr)[0] or+                     # This type will be used as a context for further inference of rvalue,+                     # we put Uninhabited if there is no information available from lvalue.+                     UninhabitedType() for sub_expr in lvalue.items]+            lvalue_type = TupleType(types, self.named_type('builtins.tuple'))+        else:+            lvalue_type = self.expr_checker.accept(lvalue)++        return lvalue_type, index_lvalue, inferred++    def is_definition(self, s: Lvalue) -> bool:+        if isinstance(s, NameExpr):+            if s.is_inferred_def:+                return True+            # If the node type is not defined, this must the first assignment+            # that we process => this is a definition, even though the semantic+            # analyzer did not recognize this as such. This can arise in code+            # that uses isinstance checks, if type checking of the primary+            # definition is skipped due to an always False type check.+            node = s.node+            if isinstance(node, Var):+                return node.type is None+        elif isinstance(s, MemberExpr):+            return s.is_inferred_def+        return False++    def infer_variable_type(self, name: Var, lvalue: Lvalue,+                            init_type: Type, context: Context) -> None:+        """Infer the type of initialized variables from initializer type."""+        if isinstance(init_type, DeletedType):+            self.msg.deleted_as_rvalue(init_type, context)+        elif not is_valid_inferred_type(init_type) and not self.no_partial_types:+            # We cannot use the type of the initialization expression for full type+            # inference (it's not specific enough), but we might be able to give+            # partial type which will be made more specific later. A partial type+            # gets generated in assignment like 'x = []' where item type is not known.+            if not self.infer_partial_type(name, lvalue, init_type):+                self.msg.need_annotation_for_var(name, context)+                self.set_inference_error_fallback_type(name, lvalue, init_type, context)+        elif (isinstance(lvalue, MemberExpr) and self.inferred_attribute_types is not None+              and lvalue.def_var and lvalue.def_var in self.inferred_attribute_types+              and not is_same_type(self.inferred_attribute_types[lvalue.def_var], init_type)):+            # Multiple, inconsistent types inferred for an attribute.+            self.msg.need_annotation_for_var(name, context)+            name.type = AnyType(TypeOfAny.from_error)+        else:+            # Infer type of the target.++            # Make the type more general (strip away function names etc.).+            init_type = strip_type(init_type)++            self.set_inferred_type(name, lvalue, init_type)++    def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool:+        if isinstance(init_type, NoneTyp):+            partial_type = PartialType(None, name, [init_type])+        elif isinstance(init_type, Instance):+            fullname = init_type.type.fullname()+            if (isinstance(lvalue, (NameExpr, MemberExpr)) and+                    (fullname == 'builtins.list' or+                     fullname == 'builtins.set' or+                     fullname == 'builtins.dict') and+                    all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)):+                partial_type = PartialType(init_type.type, name, init_type.args)+            else:+                return False+        else:+            return False+        self.set_inferred_type(name, lvalue, partial_type)+        self.partial_types[-1].map[name] = lvalue+        return True++    def set_inferred_type(self, var: Var, lvalue: Lvalue, type: Type) -> None:+        """Store inferred variable type.++        Store the type to both the variable node and the expression node that+        refers to the variable (lvalue). If var is None, do nothing.+        """+        if var and not self.current_node_deferred:+            var.type = type+            var.is_inferred = True+            if isinstance(lvalue, MemberExpr) and self.inferred_attribute_types is not None:+                # Store inferred attribute type so that we can check consistency afterwards.+                if lvalue.def_var is not None:+                    self.inferred_attribute_types[lvalue.def_var] = type+            self.store_type(lvalue, type)++    def set_inference_error_fallback_type(self, var: Var, lvalue: Lvalue, type: Type,+                                          context: Context) -> None:+        """If errors on context line are ignored, store dummy type for variable.++        If a program ignores error on type inference error, the variable should get some+        inferred type so that if can used later on in the program. Example:++          x = []  # type: ignore+          x.append(1)   # Should be ok!++        We implement this here by giving x a valid type (Any).+        """+        if context.get_line() in self.errors.ignored_lines[self.errors.file]:+            self.set_inferred_type(var, lvalue, AnyType(TypeOfAny.from_error))++    def check_simple_assignment(self, lvalue_type: Optional[Type], rvalue: Expression,+                                context: Context,+                                msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT,+                                lvalue_name: str = 'variable',+                                rvalue_name: str = 'expression') -> Type:+        if self.is_stub and isinstance(rvalue, EllipsisExpr):+            # '...' is always a valid initializer in a stub.+            return AnyType(TypeOfAny.special_form)+        else:+            always_allow_any = lvalue_type is not None and not isinstance(lvalue_type, AnyType)+            rvalue_type = self.expr_checker.accept(rvalue, lvalue_type,+                                                   always_allow_any=always_allow_any)+            if isinstance(rvalue_type, DeletedType):+                self.msg.deleted_as_rvalue(rvalue_type, context)+            if isinstance(lvalue_type, DeletedType):+                self.msg.deleted_as_lvalue(lvalue_type, context)+            elif lvalue_type:+                self.check_subtype(rvalue_type, lvalue_type, context, msg,+                                   '{} has type'.format(rvalue_name),+                                   '{} has type'.format(lvalue_name))+            return rvalue_type++    def check_member_assignment(self, instance_type: Type, attribute_type: Type,+                                rvalue: Expression, context: Context) -> Tuple[Type, bool]:+        """Type member assignment.++        This defers to check_simple_assignment, unless the member expression+        is a descriptor, in which case this checks descriptor semantics as well.++        Return the inferred rvalue_type and whether to infer anything about the attribute type.+        """+        # Descriptors don't participate in class-attribute access+        if ((isinstance(instance_type, FunctionLike) and instance_type.is_type_obj()) or+                isinstance(instance_type, TypeType)):+            rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context)+            return rvalue_type, True++        if not isinstance(attribute_type, Instance):+            rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context)+            return rvalue_type, True++        if not attribute_type.type.has_readable_member('__set__'):+            # If there is no __set__, we type-check that the assigned value matches+            # the return type of __get__. This doesn't match the python semantics,+            # (which allow you to override the descriptor with any value), but preserves+            # the type of accessing the attribute (even after the override).+            if attribute_type.type.has_readable_member('__get__'):+                attribute_type = analyze_descriptor_access(+                    instance_type, attribute_type, self.named_type,+                    self.msg, context, chk=self)+            rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context)+            return rvalue_type, True++        dunder_set = attribute_type.type.get_method('__set__')+        if dunder_set is None:+            self.msg.fail("{}.__set__ is not callable".format(attribute_type), context)+            return AnyType(TypeOfAny.from_error), False++        function = function_type(dunder_set, self.named_type('builtins.function'))+        bound_method = bind_self(function, attribute_type)+        typ = map_instance_to_supertype(attribute_type, dunder_set.info)+        dunder_set_type = expand_type_by_instance(bound_method, typ)++        _, inferred_dunder_set_type = self.expr_checker.check_call(+            dunder_set_type, [TempNode(instance_type), rvalue],+            [nodes.ARG_POS, nodes.ARG_POS], context)++        if not isinstance(inferred_dunder_set_type, CallableType):+            self.fail("__set__ is not callable", context)+            return AnyType(TypeOfAny.from_error), True++        if len(inferred_dunder_set_type.arg_types) < 2:+            # A message already will have been recorded in check_call+            return AnyType(TypeOfAny.from_error), False++        return inferred_dunder_set_type.arg_types[1], False++    def check_indexed_assignment(self, lvalue: IndexExpr,+                                 rvalue: Expression, context: Context) -> None:+        """Type check indexed assignment base[index] = rvalue.++        The lvalue argument is the base[index] expression.+        """+        self.try_infer_partial_type_from_indexed_assignment(lvalue, rvalue)+        basetype = self.expr_checker.accept(lvalue.base)+        if isinstance(basetype, TypedDictType):+            item_type = self.expr_checker.visit_typeddict_index_expr(basetype, lvalue.index)+            method_type = CallableType(+                arg_types=[self.named_type('builtins.str'), item_type],+                arg_kinds=[ARG_POS, ARG_POS],+                arg_names=[None, None],+                ret_type=NoneTyp(),+                fallback=self.named_type('builtins.function')+            )  # type: Type+        else:+            method_type = self.expr_checker.analyze_external_member_access(+                '__setitem__', basetype, context)+        lvalue.method_type = method_type+        self.expr_checker.check_call(method_type, [lvalue.index, rvalue],+                                     [nodes.ARG_POS, nodes.ARG_POS],+                                     context)++    def try_infer_partial_type_from_indexed_assignment(+            self, lvalue: IndexExpr, rvalue: Expression) -> None:+        # TODO: Should we share some of this with try_infer_partial_type?+        if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var):+            var = lvalue.base.node+            if isinstance(var.type, PartialType):+                type_type = var.type.type+                if type_type is None:+                    return  # The partial type is None.+                partial_types = self.find_partial_types(var)+                if partial_types is None:+                    return+                typename = type_type.fullname()+                if typename == 'builtins.dict':+                    # TODO: Don't infer things twice.+                    key_type = self.expr_checker.accept(lvalue.index)+                    value_type = self.expr_checker.accept(rvalue)+                    full_key_type = UnionType.make_simplified_union(+                        [key_type, var.type.inner_types[0]])+                    full_value_type = UnionType.make_simplified_union(+                        [value_type, var.type.inner_types[1]])+                    if (is_valid_inferred_type(full_key_type) and+                            is_valid_inferred_type(full_value_type)):+                        if not self.current_node_deferred:+                            var.type = self.named_generic_type('builtins.dict',+                                                               [full_key_type, full_value_type])+                            del partial_types[var]++    def visit_expression_stmt(self, s: ExpressionStmt) -> None:+        self.expr_checker.accept(s.expr, allow_none_return=True, always_allow_any=True)++    def visit_return_stmt(self, s: ReturnStmt) -> None:+        """Type check a return statement."""+        self.check_return_stmt(s)+        self.binder.unreachable()++    def check_return_stmt(self, s: ReturnStmt) -> None:+        defn = self.scope.top_function()+        if defn is not None:+            if defn.is_generator:+                return_type = self.get_generator_return_type(self.return_types[-1],+                                                             defn.is_coroutine)+            elif defn.is_coroutine:+                return_type = self.get_coroutine_return_type(self.return_types[-1])+            else:+                return_type = self.return_types[-1]++            if isinstance(return_type, UninhabitedType):+                self.fail(messages.NO_RETURN_EXPECTED, s)+                return++            if s.expr:+                is_lambda = isinstance(self.scope.top_function(), LambdaExpr)+                declared_none_return = isinstance(return_type, NoneTyp)+                declared_any_return = isinstance(return_type, AnyType)++                # This controls whether or not we allow a function call that+                # returns None as the expression of this return statement.+                # E.g. `return f()` for some `f` that returns None.  We allow+                # this only if we're in a lambda or in a function that returns+                # `None` or `Any`.+                allow_none_func_call = is_lambda or declared_none_return or declared_any_return++                # Return with a value.+                typ = self.expr_checker.accept(s.expr,+                                               return_type,+                                               allow_none_return=allow_none_func_call)++                if defn.is_async_generator:+                    self.fail("'return' with value in async generator is not allowed", s)+                    return+                # Returning a value of type Any is always fine.+                if isinstance(typ, AnyType):+                    # (Unless you asked to be warned in that case, and the+                    # function is not declared to return Any)+                    if (self.options.warn_return_any+                        and not self.current_node_deferred+                        and not is_proper_subtype(AnyType(TypeOfAny.special_form), return_type)+                        and not (defn.name() in BINARY_MAGIC_METHODS and+                                 is_literal_not_implemented(s.expr))):+                        self.msg.incorrectly_returning_any(return_type, s)+                    return++                # Disallow return expressions in functions declared to return+                # None, subject to two exceptions below.+                if declared_none_return:+                    # Lambdas are allowed to have None returns.+                    # Functions returning a value of type None are allowed to have a None return.+                    if is_lambda or isinstance(typ, NoneTyp):+                        return+                    self.fail(messages.NO_RETURN_VALUE_EXPECTED, s)+                else:+                    self.check_subtype(+                        subtype_label='got',+                        subtype=typ,+                        supertype_label='expected',+                        supertype=return_type,+                        context=s,+                        msg=messages.INCOMPATIBLE_RETURN_VALUE_TYPE)+            else:+                # Empty returns are valid in Generators with Any typed returns, but not in+                # coroutines.+                if (defn.is_generator and not defn.is_coroutine and+                        isinstance(return_type, AnyType)):+                    return++                if isinstance(return_type, (NoneTyp, AnyType)):+                    return++                if self.in_checked_function():+                    self.fail(messages.RETURN_VALUE_EXPECTED, s)++    def visit_if_stmt(self, s: IfStmt) -> None:+        """Type check an if statement."""+        # This frame records the knowledge from previous if/elif clauses not being taken.+        # Fall-through to the original frame is handled explicitly in each block.+        with self.binder.frame_context(can_skip=False, fall_through=0):+            for e, b in zip(s.expr, s.body):+                t = self.expr_checker.accept(e)++                if isinstance(t, DeletedType):+                    self.msg.deleted_as_rvalue(t, s)++                if self.options.strict_boolean:+                    is_bool = isinstance(t, Instance) and t.type.fullname() == 'builtins.bool'+                    if not (is_bool or isinstance(t, AnyType)):+                        self.fail(messages.NON_BOOLEAN_IN_CONDITIONAL, e)++                if_map, else_map = self.find_isinstance_check(e)++                # XXX Issue a warning if condition is always False?+                with self.binder.frame_context(can_skip=True, fall_through=2):+                    self.push_type_map(if_map)+                    self.accept(b)++                # XXX Issue a warning if condition is always True?+                self.push_type_map(else_map)++            with self.binder.frame_context(can_skip=False, fall_through=2):+                if s.else_body:+                    self.accept(s.else_body)++    def visit_while_stmt(self, s: WhileStmt) -> None:+        """Type check a while statement."""+        if_stmt = IfStmt([s.expr], [s.body], None)+        if_stmt.set_line(s.get_line(), s.get_column())+        self.accept_loop(if_stmt, s.else_body,+                         exit_condition=s.expr)++    def visit_operator_assignment_stmt(self,+                                       s: OperatorAssignmentStmt) -> None:+        """Type check an operator assignment statement, e.g. x += 1."""+        lvalue_type = self.expr_checker.accept(s.lvalue)+        inplace, method = infer_operator_assignment_method(lvalue_type, s.op)+        if inplace:+            # There is __ifoo__, treat as x = x.__ifoo__(y)+            rvalue_type, method_type = self.expr_checker.check_op(+                method, lvalue_type, s.rvalue, s)+            if not is_subtype(rvalue_type, lvalue_type):+                self.msg.incompatible_operator_assignment(s.op, s)+        else:+            # There is no __ifoo__, treat as x = x <foo> y+            expr = OpExpr(s.op, s.lvalue, s.rvalue)+            expr.set_line(s)+            self.check_assignment(lvalue=s.lvalue, rvalue=expr,+                                  infer_lvalue_type=True, new_syntax=False)++    def visit_assert_stmt(self, s: AssertStmt) -> None:+        self.expr_checker.accept(s.expr)++        if s.msg is not None:+            self.expr_checker.accept(s.msg)++        if isinstance(s.expr, TupleExpr) and len(s.expr.items) > 0:+            self.warn(messages.MALFORMED_ASSERT, s)++        # If this is asserting some isinstance check, bind that type in the following code+        true_map, _ = self.find_isinstance_check(s.expr)+        self.push_type_map(true_map)++    def visit_raise_stmt(self, s: RaiseStmt) -> None:+        """Type check a raise statement."""+        if s.expr:+            self.type_check_raise(s.expr, s)+        if s.from_expr:+            self.type_check_raise(s.from_expr, s, True)+        self.binder.unreachable()++    def type_check_raise(self, e: Expression, s: RaiseStmt,+                         optional: bool = False) -> None:+        typ = self.expr_checker.accept(e)+        if isinstance(typ, TypeType):+            if isinstance(typ.item, AnyType):+                return+            typ = typ.item+        if isinstance(typ, FunctionLike):+            if typ.is_type_obj():+                # Cases like "raise/from ExceptionClass".+                typeinfo = typ.type_object()+                base = self.lookup_typeinfo('builtins.BaseException')+                if base in typeinfo.mro or typeinfo.fallback_to_any:+                    # Good!+                    return+                # Else fall back to the checks below (which will fail).+        if isinstance(typ, TupleType) and self.options.python_version[0] == 2:+            # allow `raise type, value, traceback`+            # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement+            # TODO: Also check tuple item types.+            if len(typ.items) in (2, 3):+                return+        if isinstance(typ, Instance) and typ.type.fallback_to_any:+            # OK!+            return+        expected_type = self.named_type('builtins.BaseException')  # type: Type+        if optional:+            expected_type = UnionType([expected_type, NoneTyp()])+        self.check_subtype(typ, expected_type, s, messages.INVALID_EXCEPTION)++    def visit_try_stmt(self, s: TryStmt) -> None:+        """Type check a try statement."""+        # Our enclosing frame will get the result if the try/except falls through.+        # This one gets all possible states after the try block exited abnormally+        # (by exception, return, break, etc.)+        with self.binder.frame_context(can_skip=False, fall_through=0):+            # Not only might the body of the try statement exit+            # abnormally, but so might an exception handler or else+            # clause. The finally clause runs in *all* cases, so we+            # need an outer try frame to catch all intermediate states+            # in case an exception is raised during an except or else+            # clause. As an optimization, only create the outer try+            # frame when there actually is a finally clause.+            self.visit_try_without_finally(s, try_frame=bool(s.finally_body))+            if s.finally_body:+                # First we check finally_body is type safe on all abnormal exit paths+                self.accept(s.finally_body)++        if s.finally_body:+            # Then we try again for the more restricted set of options+            # that can fall through. (Why do we need to check the+            # finally clause twice? Depending on whether the finally+            # clause was reached by the try clause falling off the end+            # or exiting abnormally, after completing the finally clause+            # either flow will continue to after the entire try statement+            # or the exception/return/etc. will be processed and control+            # flow will escape. We need to check that the finally clause+            # type checks in both contexts, but only the resulting types+            # from the latter context affect the type state in the code+            # that follows the try statement.)+            self.accept(s.finally_body)++    def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None:+        """Type check a try statement, ignoring the finally block.++        On entry, the top frame should receive all flow that exits the+        try block abnormally (i.e., such that the else block does not+        execute), and its parent should receive all flow that exits+        the try block normally.+        """+        # This frame will run the else block if the try fell through.+        # In that case, control flow continues to the parent of what+        # was the top frame on entry.+        with self.binder.frame_context(can_skip=False, fall_through=2, try_frame=try_frame):+            # This frame receives exit via exception, and runs exception handlers+            with self.binder.frame_context(can_skip=False, fall_through=2):+                # Finally, the body of the try statement+                with self.binder.frame_context(can_skip=False, fall_through=2, try_frame=True):+                    self.accept(s.body)+                for i in range(len(s.handlers)):+                    with self.binder.frame_context(can_skip=True, fall_through=4):+                        typ = s.types[i]+                        if typ:+                            t = self.check_except_handler_test(typ)+                            var = s.vars[i]+                            if var:+                                # To support local variables, we make this a definition line,+                                # causing assignment to set the variable's type.+                                var.is_inferred_def = True+                                # We also temporarily set current_node_deferred to False to+                                # make sure the inference happens.+                                # TODO: Use a better solution, e.g. a+                                # separate Var for each except block.+                                am_deferring = self.current_node_deferred+                                self.current_node_deferred = False+                                self.check_assignment(var, self.temp_node(t, var))+                                self.current_node_deferred = am_deferring+                        self.accept(s.handlers[i])+                        var = s.vars[i]+                        if var:+                            # Exception variables are deleted in python 3 but not python 2.+                            # But, since it's bad form in python 2 and the type checking+                            # wouldn't work very well, we delete it anyway.++                            # Unfortunately, this doesn't let us detect usage before the+                            # try/except block.+                            if self.options.python_version[0] >= 3:+                                source = var.name+                            else:+                                source = ('(exception variable "{}", which we do not '+                                          'accept outside except: blocks even in '+                                          'python 2)'.format(var.name))+                            cast(Var, var.node).type = DeletedType(source=source)+                            self.binder.cleanse(var)+            if s.else_body:+                self.accept(s.else_body)++    def check_except_handler_test(self, n: Expression) -> Type:+        """Type check an exception handler test clause."""+        typ = self.expr_checker.accept(n)++        all_types = []  # type: List[Type]+        test_types = self.get_types_from_except_handler(typ, n)++        for ttype in test_types:+            if isinstance(ttype, AnyType):+                all_types.append(ttype)+                continue++            if isinstance(ttype, FunctionLike):+                item = ttype.items()[0]+                if not item.is_type_obj():+                    self.fail(messages.INVALID_EXCEPTION_TYPE, n)+                    return AnyType(TypeOfAny.from_error)+                exc_type = item.ret_type+            elif isinstance(ttype, TypeType):+                exc_type = ttype.item+            else:+                self.fail(messages.INVALID_EXCEPTION_TYPE, n)+                return AnyType(TypeOfAny.from_error)++            if not is_subtype(exc_type, self.named_type('builtins.BaseException')):+                self.fail(messages.INVALID_EXCEPTION_TYPE, n)+                return AnyType(TypeOfAny.from_error)++            all_types.append(exc_type)++        return UnionType.make_simplified_union(all_types)++    def get_types_from_except_handler(self, typ: Type, n: Expression) -> List[Type]:+        """Helper for check_except_handler_test to retrieve handler types."""+        if isinstance(typ, TupleType):+            return typ.items+        elif isinstance(typ, UnionType):+            return [+                union_typ+                for item in typ.relevant_items()+                for union_typ in self.get_types_from_except_handler(item, n)+            ]+        elif isinstance(typ, Instance) and is_named_instance(typ, 'builtins.tuple'):+            # variadic tuple+            return [typ.args[0]]+        else:+            return [typ]++    def visit_for_stmt(self, s: ForStmt) -> None:+        """Type check a for statement."""+        if s.is_async:+            iterator_type, item_type = self.analyze_async_iterable_item_type(s.expr)+        else:+            iterator_type, item_type = self.analyze_iterable_item_type(s.expr)+        s.inferred_item_type = item_type+        s.inferred_iterator_type = iterator_type+        self.analyze_index_variables(s.index, item_type, s.index_type is None, s)+        self.accept_loop(s.body, s.else_body)++    def analyze_async_iterable_item_type(self, expr: Expression) -> Tuple[Type, Type]:+        """Analyse async iterable expression and return iterator and iterator item types."""+        echk = self.expr_checker+        iterable = echk.accept(expr)+        method = echk.analyze_external_member_access('__aiter__', iterable, expr)+        iterator = echk.check_call(method, [], [], expr)[0]+        method = echk.analyze_external_member_access('__anext__', iterator, expr)+        awaitable = echk.check_call(method, [], [], expr)[0]+        item_type = echk.check_awaitable_expr(awaitable, expr,+                                              messages.INCOMPATIBLE_TYPES_IN_ASYNC_FOR)+        return iterator, item_type++    def analyze_iterable_item_type(self, expr: Expression) -> Tuple[Type, Type]:+        """Analyse iterable expression and return iterator and iterator item types."""+        echk = self.expr_checker+        iterable = echk.accept(expr)+        method = echk.analyze_external_member_access('__iter__', iterable, expr)+        iterator = echk.check_call(method, [], [], expr)[0]++        if isinstance(iterable, TupleType):+            joined = UninhabitedType()  # type: Type+            for item in iterable.items:+                joined = join_types(joined, item)+            return iterator, joined+        else:+            # Non-tuple iterable.+            if self.options.python_version[0] >= 3:+                nextmethod = '__next__'+            else:+                nextmethod = 'next'+            method = echk.analyze_external_member_access(nextmethod, iterator,+                                                         expr)+            return iterator, echk.check_call(method, [], [], expr)[0]++    def analyze_index_variables(self, index: Expression, item_type: Type,+                                infer_lvalue_type: bool, context: Context) -> None:+        """Type check or infer for loop or list comprehension index vars."""+        self.check_assignment(index, self.temp_node(item_type, context), infer_lvalue_type)++    def visit_del_stmt(self, s: DelStmt) -> None:+        if isinstance(s.expr, IndexExpr):+            e = s.expr+            m = MemberExpr(e.base, '__delitem__')+            m.line = s.line+            c = CallExpr(m, [e.index], [nodes.ARG_POS], [None])+            c.line = s.line+            self.expr_checker.accept(c, allow_none_return=True)+        else:+            s.expr.accept(self.expr_checker)+            for elt in flatten(s.expr):+                if isinstance(elt, NameExpr):+                    self.binder.assign_type(elt, DeletedType(source=elt.name),+                                            get_declaration(elt), False)++    def visit_decorator(self, e: Decorator) -> None:+        for d in e.decorators:+            if isinstance(d, RefExpr):+                if d.fullname == 'typing.no_type_check':+                    e.var.type = AnyType(TypeOfAny.special_form)+                    e.var.is_ready = True+                    return++        if self.recurse_into_functions:+            with self.tscope.function_scope(e.func):+                self.check_func_item(e.func, name=e.func.name())++        # Process decorators from the inside out to determine decorated signature, which+        # may be different from the declared signature.+        sig = self.function_type(e.func)  # type: Type+        for d in reversed(e.decorators):+            if refers_to_fullname(d, 'typing.overload'):+                self.fail('Single overload definition, multiple required', e)+                continue+            dec = self.expr_checker.accept(d)+            temp = self.temp_node(sig)+            fullname = None+            if isinstance(d, RefExpr):+                fullname = d.fullname+            self.check_for_untyped_decorator(e.func, dec, d)+            sig, t2 = self.expr_checker.check_call(dec, [temp],+                                                   [nodes.ARG_POS], e,+                                                   callable_name=fullname)+        self.check_untyped_after_decorator(sig, e.func)+        sig = set_callable_name(sig, e.func)+        e.var.type = sig+        e.var.is_ready = True+        if e.func.is_property:+            self.check_incompatible_property_override(e)+        if e.func.info and not e.func.is_dynamic():+            self.check_method_override(e)++    def check_for_untyped_decorator(self,+                                    func: FuncDef,+                                    dec_type: Type,+                                    dec_expr: Expression) -> None:+        if (self.options.disallow_untyped_decorators and+                is_typed_callable(func.type) and+                is_untyped_decorator(dec_type)):+            self.msg.typed_function_untyped_decorator(func.name(), dec_expr)++    def check_incompatible_property_override(self, e: Decorator) -> None:+        if not e.var.is_settable_property and e.func.info:+            name = e.func.name()+            for base in e.func.info.mro[1:]:+                base_attr = base.names.get(name)+                if not base_attr:+                    continue+                if (isinstance(base_attr.node, OverloadedFuncDef) and+                        base_attr.node.is_property and+                        cast(Decorator,+                             base_attr.node.items[0]).var.is_settable_property):+                    self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e)++    def visit_with_stmt(self, s: WithStmt) -> None:+        for expr, target in zip(s.expr, s.target):+            if s.is_async:+                self.check_async_with_item(expr, target, s.target_type is None)+            else:+                self.check_with_item(expr, target, s.target_type is None)+        self.accept(s.body)++    def check_untyped_after_decorator(self, typ: Type, func: FuncDef) -> None:+        if not self.options.disallow_any_decorated or self.is_stub:+            return++        if mypy.checkexpr.has_any_type(typ):+            self.msg.untyped_decorated_function(typ, func)++    def check_async_with_item(self, expr: Expression, target: Optional[Expression],+                              infer_lvalue_type: bool) -> None:+        echk = self.expr_checker+        ctx = echk.accept(expr)+        enter = echk.analyze_external_member_access('__aenter__', ctx, expr)+        obj = echk.check_call(enter, [], [], expr)[0]+        obj = echk.check_awaitable_expr(+            obj, expr, messages.INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AENTER)+        if target:+            self.check_assignment(target, self.temp_node(obj, expr), infer_lvalue_type)+        exit = echk.analyze_external_member_access('__aexit__', ctx, expr)+        arg = self.temp_node(AnyType(TypeOfAny.special_form), expr)+        res = echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr)[0]+        echk.check_awaitable_expr(+            res, expr, messages.INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AEXIT)++    def check_with_item(self, expr: Expression, target: Optional[Expression],+                        infer_lvalue_type: bool) -> None:+        echk = self.expr_checker+        ctx = echk.accept(expr)+        enter = echk.analyze_external_member_access('__enter__', ctx, expr)+        obj = echk.check_call(enter, [], [], expr)[0]+        if target:+            self.check_assignment(target, self.temp_node(obj, expr), infer_lvalue_type)+        exit = echk.analyze_external_member_access('__exit__', ctx, expr)+        arg = self.temp_node(AnyType(TypeOfAny.special_form), expr)+        echk.check_call(exit, [arg] * 3, [nodes.ARG_POS] * 3, expr)++    def visit_print_stmt(self, s: PrintStmt) -> None:+        for arg in s.args:+            self.expr_checker.accept(arg)+        if s.target:+            target_type = self.expr_checker.accept(s.target)+            if not isinstance(target_type, NoneTyp):+                # TODO: Also verify the type of 'write'.+                self.expr_checker.analyze_external_member_access('write', target_type, s.target)++    def visit_break_stmt(self, s: BreakStmt) -> None:+        self.binder.handle_break()++    def visit_continue_stmt(self, s: ContinueStmt) -> None:+        self.binder.handle_continue()+        return None++    def intersect_instance_callable(self, typ: Instance, callable_type: CallableType) -> Instance:+        """Creates a fake type that represents the intersection of an+        Instance and a CallableType.++        It operates by creating a bare-minimum dummy TypeInfo that+        subclasses type and adds a __call__ method matching callable_type.+        """++        # In order for this to work in incremental mode, the type we generate needs to+        # have a valid fullname and a corresponding entry in a symbol table. We generate+        # a unique name inside the symbol table of the current module.+        cur_module = cast(MypyFile, self.scope.stack[0])+        gen_name = gen_unique_name("<callable subtype of {}>".format(typ.type.name()),+                                   cur_module.names)++        # Build the fake ClassDef and TypeInfo together.+        # The ClassDef is full of lies and doesn't actually contain a body.+        # Use format_bare to generate a nice name for error messages.+        # We skip fully filling out a handful of TypeInfo fields because they+        # should be irrelevant for a generated type like this:+        # is_protocol, protocol_members, is_abstract+        short_name = self.msg.format_bare(typ)+        cdef = ClassDef(short_name, Block([]))+        cdef.fullname = cur_module.fullname() + '.' + gen_name+        info = TypeInfo(SymbolTable(), cdef, cur_module.fullname())+        cdef.info = info+        info.bases = [typ]+        calculate_mro(info)+        info.calculate_metaclass_type()++        # Build up a fake FuncDef so we can populate the symbol table.+        func_def = FuncDef('__call__', [], Block([]), callable_type)+        func_def._fullname = cdef.fullname + '.__call__'+        func_def.info = info+        info.names['__call__'] = SymbolTableNode(MDEF, func_def)++        cur_module.names[gen_name] = SymbolTableNode(GDEF, info)++        return Instance(info, [])++    def make_fake_callable(self, typ: Instance) -> Instance:+        """Produce a new type that makes type Callable with a generic callable type."""++        fallback = self.named_type('builtins.function')+        callable_type = CallableType([AnyType(TypeOfAny.explicit),+                                      AnyType(TypeOfAny.explicit)],+                                     [nodes.ARG_STAR, nodes.ARG_STAR2],+                                     [None, None],+                                     ret_type=AnyType(TypeOfAny.explicit),+                                     fallback=fallback,+                                     is_ellipsis_args=True)++        return self.intersect_instance_callable(typ, callable_type)++    def partition_by_callable(self, typ: Type,+                              unsound_partition: bool) -> Tuple[List[Type], List[Type]]:+        """Takes in a type and partitions that type into callable subtypes and+        uncallable subtypes.++        Thus, given:+        `callables, uncallables = partition_by_callable(type)`++        If we assert `callable(type)` then `type` has type Union[*callables], and+        If we assert `not callable(type)` then `type` has type Union[*uncallables]++        If unsound_partition is set, assume that anything that is not+        clearly callable is in fact not callable. Otherwise we generate a+        new subtype that *is* callable.++        Guaranteed to not return [], []++        """+        if isinstance(typ, FunctionLike) or isinstance(typ, TypeType):+            return [typ], []++        if isinstance(typ, AnyType):+            return [typ], [typ]++        if isinstance(typ, UnionType):+            callables = []+            uncallables = []+            for subtype in typ.relevant_items():+                # Use unsound_partition when handling unions in order to+                # allow the expected type discrimination.+                subcallables, subuncallables = self.partition_by_callable(subtype,+                                                                          unsound_partition=True)+                callables.extend(subcallables)+                uncallables.extend(subuncallables)+            return callables, uncallables++        if isinstance(typ, TypeVarType):+            # We could do better probably?+            # Refine the the type variable's bound as our type in the case that+            # callable() is true. This unfortunately loses the information that+            # the type is a type variable in that branch.+            # This matches what is done for isinstance, but it may be possible to+            # do better.+            # If it is possible for the false branch to execute, return the original+            # type to avoid losing type information.+            callables, uncallables = self.partition_by_callable(typ.erase_to_union_or_bound(),+                                                                unsound_partition)+            uncallables = [typ] if len(uncallables) else []+            return callables, uncallables++        # A TupleType is callable if its fallback is, but needs special handling+        # when we dummy up a new type.+        ityp = typ+        if isinstance(typ, TupleType):+            ityp = typ.fallback++        if isinstance(ityp, Instance):+            method = ityp.type.get_method('__call__')+            if method and method.type:+                callables, uncallables = self.partition_by_callable(method.type,+                                                                    unsound_partition=False)+                if len(callables) and not len(uncallables):+                    # Only consider the type callable if its __call__ method is+                    # definitely callable.+                    return [typ], []++            if not unsound_partition:+                fake = self.make_fake_callable(ityp)+                if isinstance(typ, TupleType):+                    fake.type.tuple_type = TupleType(typ.items, fake)+                    return [fake.type.tuple_type], [typ]+                return [fake], [typ]++        if unsound_partition:+            return [], [typ]+        else:+            # We don't know how properly make the type callable.+            return [typ], [typ]++    def conditional_callable_type_map(self, expr: Expression,+                                      current_type: Optional[Type],+                                      ) -> Tuple[TypeMap, TypeMap]:+        """Takes in an expression and the current type of the expression.++        Returns a 2-tuple: The first element is a map from the expression to+        the restricted type if it were callable. The second element is a+        map from the expression to the type it would hold if it weren't+        callable.+        """+        if not current_type:+            return {}, {}++        if isinstance(current_type, AnyType):+            return {}, {}++        callables, uncallables = self.partition_by_callable(current_type,+                                                            unsound_partition=False)++        if len(callables) and len(uncallables):+            callable_map = {expr: UnionType.make_union(callables)} if len(callables) else None+            uncallable_map = {+                expr: UnionType.make_union(uncallables)} if len(uncallables) else None+            return callable_map, uncallable_map++        elif len(callables):+            return {}, None++        return None, {}++    def find_isinstance_check(self, node: Expression+                              ) -> Tuple[TypeMap, TypeMap]:+        """Find any isinstance checks (within a chain of ands).  Includes+        implicit and explicit checks for None and calls to callable.++        Return value is a map of variables to their types if the condition+        is true and a map of variables to their types if the condition is false.++        If either of the values in the tuple is None, then that particular+        branch can never occur.++        Guaranteed to not return None, None. (But may return {}, {})+        """+        type_map = self.type_map+        if is_true_literal(node):+            return {}, None+        elif is_false_literal(node):+            return None, {}+        elif isinstance(node, CallExpr):+            if refers_to_fullname(node.callee, 'builtins.isinstance'):+                if len(node.args) != 2:  # the error will be reported later+                    return {}, {}+                expr = node.args[0]+                if literal(expr) == LITERAL_TYPE:+                    vartype = type_map[expr]+                    type = get_isinstance_type(node.args[1], type_map)+                    return conditional_type_map(expr, vartype, type)+            elif refers_to_fullname(node.callee, 'builtins.issubclass'):+                expr = node.args[0]+                if literal(expr) == LITERAL_TYPE:+                    vartype = type_map[expr]+                    type = get_isinstance_type(node.args[1], type_map)+                    if isinstance(vartype, UnionType):+                        union_list = []+                        for t in vartype.items:+                            if isinstance(t, TypeType):+                                union_list.append(t.item)+                            else:+                                #  this is an error that should be reported earlier+                                #  if we reach here, we refuse to do any type inference+                                return {}, {}+                        vartype = UnionType(union_list)+                    elif isinstance(vartype, TypeType):+                        vartype = vartype.item+                    else:+                        # any other object whose type we don't know precisely+                        # for example, Any or Instance of type type+                        return {}, {}  # unknown type+                    yes_map, no_map = conditional_type_map(expr, vartype, type)+                    yes_map, no_map = map(convert_to_typetype, (yes_map, no_map))+                    return yes_map, no_map+            elif refers_to_fullname(node.callee, 'builtins.callable'):+                expr = node.args[0]+                if literal(expr) == LITERAL_TYPE:+                    vartype = type_map[expr]+                    return self.conditional_callable_type_map(expr, vartype)+        elif isinstance(node, ComparisonExpr) and experiments.STRICT_OPTIONAL:+            # Check for `x is None` and `x is not None`.+            is_not = node.operators == ['is not']+            if any(is_literal_none(n) for n in node.operands) and (+                    is_not or node.operators == ['is']):+                if_vars = {}  # type: TypeMap+                else_vars = {}  # type: TypeMap+                for expr in node.operands:+                    if (literal(expr) == LITERAL_TYPE and not is_literal_none(expr)+                            and expr in type_map):+                        # This should only be true at most once: there should be+                        # two elements in node.operands, and at least one of them+                        # should represent a None.+                        vartype = type_map[expr]+                        none_typ = [TypeRange(NoneTyp(), is_upper_bound=False)]+                        if_vars, else_vars = conditional_type_map(expr, vartype, none_typ)+                        break++                if is_not:+                    if_vars, else_vars = else_vars, if_vars+                return if_vars, else_vars+            # Check for `x == y` where x is of type Optional[T] and y is of type T+            # or a type that overlaps with T (or vice versa).+            elif node.operators == ['==']:+                first_type = type_map[node.operands[0]]+                second_type = type_map[node.operands[1]]+                if is_optional(first_type) != is_optional(second_type):+                    if is_optional(first_type):+                        optional_type, comp_type = first_type, second_type+                        optional_expr = node.operands[0]+                    else:+                        optional_type, comp_type = second_type, first_type+                        optional_expr = node.operands[1]+                    if is_overlapping_types(optional_type, comp_type):+                        return {optional_expr: remove_optional(optional_type)}, {}+            elif node.operators in [['in'], ['not in']]:+                expr = node.operands[0]+                left_type = type_map[expr]+                right_type = builtin_item_type(type_map[node.operands[1]])+                right_ok = right_type and (not is_optional(right_type) and+                                           (not isinstance(right_type, Instance) or+                                            right_type.type.fullname() != 'builtins.object'))+                if (right_type and right_ok and is_optional(left_type) and+                        literal(expr) == LITERAL_TYPE and not is_literal_none(expr) and+                        is_overlapping_types(left_type, right_type)):+                    if node.operators == ['in']:+                        return {expr: remove_optional(left_type)}, {}+                    if node.operators == ['not in']:+                        return {}, {expr: remove_optional(left_type)}+        elif isinstance(node, RefExpr):+            # Restrict the type of the variable to True-ish/False-ish in the if and else branches+            # respectively+            vartype = type_map[node]+            if_type = true_only(vartype)+            else_type = false_only(vartype)+            ref = node  # type: Expression+            if_map = {ref: if_type} if not isinstance(if_type, UninhabitedType) else None+            else_map = {ref: else_type} if not isinstance(else_type, UninhabitedType) else None+            return if_map, else_map+        elif isinstance(node, OpExpr) and node.op == 'and':+            left_if_vars, left_else_vars = self.find_isinstance_check(node.left)+            right_if_vars, right_else_vars = self.find_isinstance_check(node.right)++            # (e1 and e2) is true if both e1 and e2 are true,+            # and false if at least one of e1 and e2 is false.+            return (and_conditional_maps(left_if_vars, right_if_vars),+                    or_conditional_maps(left_else_vars, right_else_vars))+        elif isinstance(node, OpExpr) and node.op == 'or':+            left_if_vars, left_else_vars = self.find_isinstance_check(node.left)+            right_if_vars, right_else_vars = self.find_isinstance_check(node.right)++            # (e1 or e2) is true if at least one of e1 or e2 is true,+            # and false if both e1 and e2 are false.+            return (or_conditional_maps(left_if_vars, right_if_vars),+                    and_conditional_maps(left_else_vars, right_else_vars))+        elif isinstance(node, UnaryExpr) and node.op == 'not':+            left, right = self.find_isinstance_check(node.expr)+            return right, left++        # Not a supported isinstance check+        return {}, {}++    #+    # Helpers+    #++    def check_subtype(self, subtype: Type, supertype: Type, context: Context,+                      msg: str = messages.INCOMPATIBLE_TYPES,+                      subtype_label: Optional[str] = None,+                      supertype_label: Optional[str] = None) -> bool:+        """Generate an error if the subtype is not compatible with+        supertype."""+        if is_subtype(subtype, supertype):+            return True+        else:+            if self.should_suppress_optional_error([subtype]):+                return False+            extra_info = []  # type: List[str]+            note_msg = ''+            if subtype_label is not None or supertype_label is not None:+                subtype_str, supertype_str = self.msg.format_distinctly(subtype, supertype)+                if subtype_label is not None:+                    extra_info.append(subtype_label + ' ' + subtype_str)+                if supertype_label is not None:+                    extra_info.append(supertype_label + ' ' + supertype_str)+                note_msg = make_inferred_type_note(context, subtype,+                                                   supertype, supertype_str)+            if extra_info:+                msg += ' (' + ', '.join(extra_info) + ')'+            self.fail(msg, context)+            if note_msg:+                self.note(note_msg, context)+            if (isinstance(supertype, Instance) and supertype.type.is_protocol and+                    isinstance(subtype, (Instance, TupleType, TypedDictType))):+                self.msg.report_protocol_problems(subtype, supertype, context)+            if isinstance(supertype, CallableType) and isinstance(subtype, Instance):+                call = find_member('__call__', subtype, subtype)+                if call:+                    self.msg.note_call(subtype, call, context)+            if isinstance(subtype, (CallableType, Overloaded)) and isinstance(supertype, Instance):+                if supertype.type.is_protocol and supertype.type.protocol_members == ['__call__']:+                    call = find_member('__call__', supertype, subtype)+                    assert call is not None+                    self.msg.note_call(supertype, call, context)+            return False++    def contains_none(self, t: Type) -> bool:+        return (+            isinstance(t, NoneTyp) or+            (isinstance(t, UnionType) and any(self.contains_none(ut) for ut in t.items)) or+            (isinstance(t, TupleType) and any(self.contains_none(tt) for tt in t.items)) or+            (isinstance(t, Instance) and bool(t.args)+             and any(self.contains_none(it) for it in t.args))+        )++    def should_suppress_optional_error(self, related_types: List[Type]) -> bool:+        return self.suppress_none_errors and any(self.contains_none(t) for t in related_types)++    def named_type(self, name: str) -> Instance:+        """Return an instance type with type given by the name and no+        type arguments. For example, named_type('builtins.object')+        produces the object type.+        """+        # Assume that the name refers to a type.+        sym = self.lookup_qualified(name)+        node = sym.node+        if isinstance(node, TypeAlias):+            assert isinstance(node.target, Instance)+            node = node.target.type+        assert isinstance(node, TypeInfo)+        any_type = AnyType(TypeOfAny.from_omitted_generics)+        return Instance(node, [any_type] * len(node.defn.type_vars))++    def named_generic_type(self, name: str, args: List[Type]) -> Instance:+        """Return an instance with the given name and type arguments.++        Assume that the number of arguments is correct.  Assume that+        the name refers to a compatible generic type.+        """+        info = self.lookup_typeinfo(name)+        # TODO: assert len(args) == len(info.defn.type_vars)+        return Instance(info, args)++    def lookup_typeinfo(self, fullname: str) -> TypeInfo:+        # Assume that the name refers to a class.+        sym = self.lookup_qualified(fullname)+        node = sym.node+        assert isinstance(node, TypeInfo)+        return node++    def type_type(self) -> Instance:+        """Return instance type 'type'."""+        return self.named_type('builtins.type')++    def str_type(self) -> Instance:+        """Return instance type 'str'."""+        return self.named_type('builtins.str')++    def store_type(self, node: Expression, typ: Type) -> None:+        """Store the type of a node in the type map."""+        self.type_map[node] = typ++    def in_checked_function(self) -> bool:+        """Should we type-check the current function?++        - Yes if --check-untyped-defs is set.+        - Yes outside functions.+        - Yes in annotated functions.+        - No otherwise.+        """+        return (self.options.check_untyped_defs+                or not self.dynamic_funcs+                or not self.dynamic_funcs[-1])++    def lookup(self, name: str, kind: int) -> SymbolTableNode:+        """Look up a definition from the symbol table with the given name.+        TODO remove kind argument+        """+        if name in self.globals:+            return self.globals[name]+        else:+            b = self.globals.get('__builtins__', None)+            if b:+                table = cast(MypyFile, b.node).names+                if name in table:+                    return table[name]+            raise KeyError('Failed lookup: {}'.format(name))++    def lookup_qualified(self, name: str) -> SymbolTableNode:+        if '.' not in name:+            return self.lookup(name, GDEF)  # FIX kind+        else:+            parts = name.split('.')+            n = self.modules[parts[0]]+            for i in range(1, len(parts) - 1):+                sym = n.names.get(parts[i])+                assert sym is not None, "Internal error: attempted lookup of unknown name"+                n = cast(MypyFile, sym.node)+            last = parts[-1]+            if last in n.names:+                return n.names[last]+            elif len(parts) == 2 and parts[0] == 'builtins':+                raise KeyError("Could not find builtin symbol '{}'. (Are you running a "+                               "test case? If so, make sure to include a fixture that "+                               "defines this symbol.)".format(last))+            else:+                msg = "Failed qualified lookup: '{}' (fullname = '{}')."+                raise KeyError(msg.format(last, name))++    @contextmanager+    def enter_partial_types(self, *, is_function: bool = False,+                            is_class: bool = False) -> Iterator[None]:+        """Enter a new scope for collecting partial types.++        Also report errors for (some) variables which still have partial+        types, i.e. we couldn't infer a complete type.+        """+        self.partial_types.append(PartialTypeScope({}, is_function))+        yield++        partial_types, _ = self.partial_types.pop()+        if not self.current_node_deferred:+            for var, context in partial_types.items():+                # If we require local partial types, there are a few exceptions where+                # we fall back to inferring just "None" as the type from a None initializer:+                #+                # 1. If all happens within a single function this is acceptable, since only+                #    the topmost function is a separate target in fine-grained incremental mode.+                #    We primarily want to avoid "splitting" partial types across targets.+                #+                # 2. A None initializer in the class body if the attribute is defined in a base+                #    class is fine, since the attribute is already defined and it's currently okay+                #    to vary the type of an attribute covariantly. The None type will still be+                #    checked for compatibility with base classes elsewhere. Without this exception+                #    mypy could require an annotation for an attribute that already has been+                #    declared in a base class, which would be bad.+                allow_none = (not self.options.local_partial_types+                              or is_function+                              or (is_class and self.is_defined_in_base_class(var)))+                if (allow_none+                        and isinstance(var.type, PartialType)+                        and var.type.type is None):+                    var.type = NoneTyp()+                else:+                    if var not in self.partial_reported:+                        self.msg.need_annotation_for_var(var, context)+                        self.partial_reported.add(var)+                    # Give the variable an 'Any' type to avoid generating multiple errors+                    # from a single missing annotation.+                    var.type = AnyType(TypeOfAny.from_error)++    def is_defined_in_base_class(self, var: Var) -> bool:+        if var.info:+            for base in var.info.mro[1:]:+                if base.get(var.name()) is not None:+                    return True+            if var.info.fallback_to_any:+                return True+        return False++    def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]:+        """Look for an active partial type scope containing variable.++        A scope is active if assignments in the current context can refine a partial+        type originally defined in the scope. This is affected by the local_partial_types+        configuration option.+        """+        in_scope, partial_types = self.find_partial_types_in_all_scopes(var)+        if in_scope:+            return partial_types+        return None++    def find_partial_types_in_all_scopes(self, var: Var) -> Tuple[bool,+                                                                  Optional[Dict[Var, Context]]]:+        """Look for partial type scope containing variable.++        Return tuple (is the scope active, scope).+        """+        active = self.partial_types+        inactive = []  # type: List[PartialTypeScope]+        if self.options.local_partial_types:+            # All scopes within the outermost function are active. Scopes out of+            # the outermost function are inactive to allow local reasoning (important+            # for fine-grained incremental mode).+            for i, t in enumerate(self.partial_types):+                if t.is_function:+                    active = self.partial_types[i:]+                    inactive = self.partial_types[:i]+                    break+            else:+                # Not within a function -- only the innermost scope is in scope.+                active = self.partial_types[-1:]+                inactive = self.partial_types[:-1]+        # First look within in-scope partial types.+        for scope in reversed(active):+            if var in scope.map:+                return True, scope.map+        # Then for out-of-scope partial types.+        for scope in reversed(inactive):+            if var in scope.map:+                return False, scope.map+        return False, None++    def temp_node(self, t: Type, context: Optional[Context] = None) -> TempNode:+        """Create a temporary node with the given, fixed type."""+        temp = TempNode(t)+        if context:+            temp.set_line(context.get_line())+        return temp++    def fail(self, msg: str, context: Context) -> None:+        """Produce an error message."""+        self.msg.fail(msg, context)++    def warn(self, msg: str, context: Context) -> None:+        """Produce a warning message."""+        self.msg.warn(msg, context)++    def note(self, msg: str, context: Context, offset: int = 0) -> None:+        """Produce a note."""+        self.msg.note(msg, context, offset=offset)++    def iterable_item_type(self, instance: Instance) -> Type:+        iterable = map_instance_to_supertype(+            instance,+            self.lookup_typeinfo('typing.Iterable'))+        item_type = iterable.args[0]+        if not isinstance(item_type, AnyType):+            # This relies on 'map_instance_to_supertype' returning 'Iterable[Any]'+            # in case there is no explicit base class.+            return item_type+        # Try also structural typing.+        iter_type = find_member('__iter__', instance, instance)+        if (iter_type and isinstance(iter_type, CallableType) and+                isinstance(iter_type.ret_type, Instance)):+            iterator = map_instance_to_supertype(iter_type.ret_type,+                                                 self.lookup_typeinfo('typing.Iterator'))+            item_type = iterator.args[0]+        return item_type++    def function_type(self, func: FuncBase) -> FunctionLike:+        return function_type(func, self.named_type('builtins.function'))++    def push_type_map(self, type_map: 'TypeMap') -> None:+        if type_map is None:+            self.binder.unreachable()+        else:+            for expr, type in type_map.items():+                self.binder.put(expr, type)+++def conditional_type_map(expr: Expression,+                         current_type: Optional[Type],+                         proposed_type_ranges: Optional[List[TypeRange]],+                         ) -> Tuple[TypeMap, TypeMap]:+    """Takes in an expression, the current type of the expression, and a+    proposed type of that expression.++    Returns a 2-tuple: The first element is a map from the expression to+    the proposed type, if the expression can be the proposed type.  The+    second element is a map from the expression to the type it would hold+    if it was not the proposed type, if any. None means bot, {} means top"""+    if proposed_type_ranges:+        if len(proposed_type_ranges) == 1:+            proposed_type = proposed_type_ranges[0].item  # Union with a single type breaks tests+        else:+            proposed_type = UnionType([type_range.item for type_range in proposed_type_ranges])+        if current_type:+            if (not any(type_range.is_upper_bound for type_range in proposed_type_ranges)+               and is_proper_subtype(current_type, proposed_type)):+                # Expression is always of one of the types in proposed_type_ranges+                return {}, None+            elif not is_overlapping_types(current_type, proposed_type):+                # Expression is never of any type in proposed_type_ranges+                return None, {}+            else:+                # we can only restrict when the type is precise, not bounded+                proposed_precise_type = UnionType([type_range.item+                                          for type_range in proposed_type_ranges+                                          if not type_range.is_upper_bound])+                remaining_type = restrict_subtype_away(current_type, proposed_precise_type)+                return {expr: proposed_type}, {expr: remaining_type}+        else:+            return {expr: proposed_type}, {}+    else:+        # An isinstance check, but we don't understand the type+        return {}, {}+++def gen_unique_name(base: str, table: SymbolTable) -> str:+    """Generate a name that does not appear in table by appending numbers to base."""+    if base not in table:+        return base+    i = 1+    while base + str(i) in table:+        i += 1+    return base + str(i)+++def is_true_literal(n: Expression) -> bool:+    return (refers_to_fullname(n, 'builtins.True')+            or isinstance(n, IntExpr) and n.value == 1)+++def is_false_literal(n: Expression) -> bool:+    return (refers_to_fullname(n, 'builtins.False')+            or isinstance(n, IntExpr) and n.value == 0)+++def is_literal_none(n: Expression) -> bool:+    return isinstance(n, NameExpr) and n.fullname == 'builtins.None'+++def is_optional(t: Type) -> bool:+    return isinstance(t, UnionType) and any(isinstance(e, NoneTyp) for e in t.items)+++def remove_optional(typ: Type) -> Type:+    if isinstance(typ, UnionType):+        return UnionType.make_union([t for t in typ.items if not isinstance(t, NoneTyp)])+    else:+        return typ+++def is_literal_not_implemented(n: Expression) -> bool:+    return isinstance(n, NameExpr) and n.fullname == 'builtins.NotImplemented'+++def builtin_item_type(tp: Type) -> Optional[Type]:+    """Get the item type of a builtin container.++    If 'tp' is not one of the built containers (these includes NamedTuple and TypedDict)+    or if the container is not parameterized (like List or List[Any])+    return None. This function is used to narrow optional types in situations like this:++        x: Optional[int]+        if x in (1, 2, 3):+            x + 42  # OK++    Note: this is only OK for built-in containers, where we know the behavior+    of __contains__.+    """+    if isinstance(tp, Instance):+        if tp.type.fullname() in ['builtins.list', 'builtins.tuple', 'builtins.dict',+                                  'builtins.set', 'builtins.frozenset']:+            if not tp.args:+                # TODO: fix tuple in lib-stub/builtins.pyi (it should be generic).+                return None+            if not isinstance(tp.args[0], AnyType):+                return tp.args[0]+    elif isinstance(tp, TupleType) and all(not isinstance(it, AnyType) for it in tp.items):+        return UnionType.make_simplified_union(tp.items)  # this type is not externally visible+    elif isinstance(tp, TypedDictType):+        # TypedDict always has non-optional string keys.+        if tp.fallback.type.fullname() == 'typing.Mapping':+            return tp.fallback.args[0]+        elif tp.fallback.type.bases[0].type.fullname() == 'typing.Mapping':+            return tp.fallback.type.bases[0].args[0]+    return None+++def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap:+    """Calculate what information we can learn from the truth of (e1 and e2)+    in terms of the information that we can learn from the truth of e1 and+    the truth of e2.+    """++    if m1 is None or m2 is None:+        # One of the conditions can never be true.+        return None+    # Both conditions can be true; combine the information. Anything+    # we learn from either conditions's truth is valid. If the same+    # expression's type is refined by both conditions, we somewhat+    # arbitrarily give precedence to m2. (In the future, we could use+    # an intersection type.)+    result = m2.copy()+    m2_keys = set(literal_hash(n2) for n2 in m2)+    for n1 in m1:+        if literal_hash(n1) not in m2_keys:+            result[n1] = m1[n1]+    return result+++def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap:+    """Calculate what information we can learn from the truth of (e1 or e2)+    in terms of the information that we can learn from the truth of e1 and+    the truth of e2.+    """++    if m1 is None:+        return m2+    if m2 is None:+        return m1+    # Both conditions can be true. Combine information about+    # expressions whose type is refined by both conditions. (We do not+    # learn anything about expressions whose type is refined by only+    # one condition.)+    result = {}+    for n1 in m1:+        for n2 in m2:+            if literal_hash(n1) == literal_hash(n2):+                result[n1] = UnionType.make_simplified_union([m1[n1], m2[n2]])+    return result+++def convert_to_typetype(type_map: TypeMap) -> TypeMap:+    converted_type_map = {}  # type: Dict[Expression, Type]+    if type_map is None:+        return None+    for expr, typ in type_map.items():+        if not isinstance(typ, (UnionType, Instance)):+            # unknown type; error was likely reported earlier+            return {}+        converted_type_map[expr] = TypeType.make_normalized(typ)+    return converted_type_map+++def flatten(t: Expression) -> List[Expression]:+    """Flatten a nested sequence of tuples/lists into one list of nodes."""+    if isinstance(t, TupleExpr) or isinstance(t, ListExpr):+        return [b for a in t.items for b in flatten(a)]+    else:+        return [t]+++def flatten_types(t: Type) -> List[Type]:+    """Flatten a nested sequence of tuples into one list of nodes."""+    if isinstance(t, TupleType):+        return [b for a in t.items for b in flatten_types(a)]+    else:+        return [t]+++def get_isinstance_type(expr: Expression,+                        type_map: Dict[Expression, Type]) -> Optional[List[TypeRange]]:+    all_types = flatten_types(type_map[expr])+    types = []  # type: List[TypeRange]+    for typ in all_types:+        if isinstance(typ, FunctionLike) and typ.is_type_obj():+            # Type variables may be present -- erase them, which is the best+            # we can do (outside disallowing them here).+            typ = erase_typevars(typ.items()[0].ret_type)+            types.append(TypeRange(typ, is_upper_bound=False))+        elif isinstance(typ, TypeType):+            # Type[A] means "any type that is a subtype of A" rather than "precisely type A"+            # we indicate this by setting is_upper_bound flag+            types.append(TypeRange(typ.item, is_upper_bound=True))+        elif isinstance(typ, Instance) and typ.type.fullname() == 'builtins.type':+            object_type = Instance(typ.type.mro[-1], [])+            types.append(TypeRange(object_type, is_upper_bound=True))+        elif isinstance(typ, AnyType):+            types.append(TypeRange(typ, is_upper_bound=False))+        else:  # we didn't see an actual type, but rather a variable whose value is unknown to us+            return None+    if not types:+        # this can happen if someone has empty tuple as 2nd argument to isinstance+        # strictly speaking, we should return UninhabitedType but for simplicity we will simply+        # refuse to do any type inference for now+        return None+    return types+++def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem:+    visitor = TypeTransformVisitor(map)+    ret = defn.accept(visitor)+    assert isinstance(ret, FuncItem)+    return ret+++class TypeTransformVisitor(TransformVisitor):+    def __init__(self, map: Dict[TypeVarId, Type]) -> None:+        super().__init__()+        self.map = map++    def type(self, type: Type) -> Type:+        return expand_type(type, self.map)+++def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool:+    """Can a single call match both t and s, based just on positional argument counts?+    """+    min_args = max(t.min_args, s.min_args)+    max_args = min(t.max_possible_positional_args(), s.max_possible_positional_args())+    return min_args <= max_args+++def is_unsafe_overlapping_overload_signatures(signature: CallableType,+                                              other: CallableType) -> bool:+    """Check if two overloaded function signatures may be unsafely overlapping.++    We consider two functions 's' and 't' to be unsafely overlapping both if+    of the following are true:++    1.  s's parameters are all more precise or partially overlapping with t's+    2.  s's return type is NOT a subtype of t's.++    Assumes that 'signature' appears earlier in the list of overload+    alternatives then 'other' and that their argument counts are overlapping.+    """+    # TODO: Handle partially overlapping parameter types+    #+    # For example, the signatures "f(x: Union[A, B]) -> int" and "f(x: Union[B, C]) -> str"+    # is unsafe: the parameter types are partially overlapping.+    #+    # To fix this, we need to either modify meet.is_overlapping_types or add a new+    # function and use "is_more_precise(...) or is_partially_overlapping(...)" for the is_compat+    # checks.+    #+    # (We already have a rudimentary implementation of 'is_partially_overlapping', but it only+    # attempts to handle the obvious cases -- see its docstring for more info.)++    def is_more_precise_or_partially_overlapping(t: Type, s: Type) -> bool:+        return is_more_precise(t, s) or is_partially_overlapping_types(t, s)++    return is_callable_compatible(signature, other,+                                  is_compat=is_more_precise_or_partially_overlapping,+                                  is_compat_return=lambda l, r: not is_subtype(l, r),+                                  check_args_covariantly=True,+                                  allow_partial_overlap=True)+++def overload_can_never_match(signature: CallableType, other: CallableType) -> bool:+    """Check if the 'other' method can never be matched due to 'signature'.++    This can happen if signature's parameters are all strictly broader then+    other's parameters.++    Assumes that both signatures have overlapping argument counts.+    """+    return is_callable_compatible(signature, other,+                                  is_compat=is_more_precise,+                                  ignore_return=True)+++def is_unsafe_overlapping_operator_signatures(signature: Type, other: Type) -> bool:+    """Check if two operator method signatures may be unsafely overlapping.++    Two signatures s and t are overlapping if both can be valid for the same+    statically typed values and the return types are incompatible.++    Assume calls are first checked against 'signature', then against 'other'.+    Thus if 'signature' is more general than 'other', there is no unsafe+    overlapping.++    TODO: Clean up this function and make it not perform type erasure.++    Context: This function was previously used to make sure both overloaded+    functions and operator methods were not unsafely overlapping.++    We changed the semantics for we should handle overloaded definitions,+    but not operator functions. (We can't reuse the same semantics for both:+    the overload semantics are too restrictive here).++    We should rewrite this method so that:++    1.  It uses many of the improvements made to overloads: in particular,+        eliminating type erasure.++    2.  It contains just the logic necessary for operator methods.+    """+    if isinstance(signature, CallableType):+        if isinstance(other, CallableType):+            # TODO varargs+            # TODO keyword args+            # TODO erasure+            # TODO allow to vary covariantly+            # Check if the argument counts are overlapping.+            min_args = max(signature.min_args, other.min_args)+            max_args = min(len(signature.arg_types), len(other.arg_types))+            if min_args > max_args:+                # Argument counts are not overlapping.+                return False+            # Signatures are overlapping iff if they are overlapping for the+            # smallest common argument count.+            for i in range(min_args):+                t1 = signature.arg_types[i]+                t2 = other.arg_types[i]+                if not is_overlapping_types(t1, t2):+                    return False+            # All arguments types for the smallest common argument count are+            # overlapping => the signature is overlapping. The overlapping is+            # safe if the return types are identical.+            if is_same_type(signature.ret_type, other.ret_type):+                return False+            # If the first signature has more general argument types, the+            # latter will never be called+            if is_more_general_arg_prefix(signature, other):+                return False+            # Special case: all args are subtypes, and returns are subtypes+            if (all(is_proper_subtype(s, o)+                    for (s, o) in zip(signature.arg_types, other.arg_types)) and+                    is_subtype(signature.ret_type, other.ret_type)):+                return False+            return not is_more_precise_signature(signature, other)+    return True+++def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool:+    """Does t have wider arguments than s?"""+    # TODO should an overload with additional items be allowed to be more+    #      general than one with fewer items (or just one item)?+    if isinstance(t, CallableType):+        if isinstance(s, CallableType):+            return is_callable_compatible(t, s,+                                          is_compat=is_proper_subtype,+                                          ignore_return=True)+    elif isinstance(t, FunctionLike):+        if isinstance(s, FunctionLike):+            if len(t.items()) == len(s.items()):+                return all(is_same_arg_prefix(items, itemt)+                           for items, itemt in zip(t.items(), s.items()))+    return False+++def is_equivalent_type_var_def(tv1: TypeVarDef, tv2: TypeVarDef) -> bool:+    """Are type variable definitions equivalent?++    Ignore ids, locations in source file and names.+    """+    return (+        tv1.variance == tv2.variance+        and is_same_types(tv1.values, tv2.values)+        and ((tv1.upper_bound is None and tv2.upper_bound is None)+             or (tv1.upper_bound is not None+                 and tv2.upper_bound is not None+                 and is_same_type(tv1.upper_bound, tv2.upper_bound))))+++def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool:+    return is_callable_compatible(t, s,+                                  is_compat=is_same_type,+                                  ignore_return=True,+                                  check_args_covariantly=True,+                                  ignore_pos_arg_names=True)+++def is_more_precise_signature(t: CallableType, s: CallableType) -> bool:+    """Is t more precise than s?+    A signature t is more precise than s if all argument types and the return+    type of t are more precise than the corresponding types in s.+    Assume that the argument kinds and names are compatible, and that the+    argument counts are overlapping.+    """+    # TODO generic function types+    # Only consider the common prefix of argument types.+    for argt, args in zip(t.arg_types, s.arg_types):+        if not is_more_precise(argt, args):+            return False+    return is_more_precise(t.ret_type, s.ret_type)+++def infer_operator_assignment_method(typ: Type, operator: str) -> Tuple[bool, str]:+    """Determine if operator assignment on given value type is in-place, and the method name.++    For example, if operator is '+', return (True, '__iadd__') or (False, '__add__')+    depending on which method is supported by the type.+    """+    method = nodes.op_methods[operator]+    if isinstance(typ, Instance):+        if operator in nodes.ops_with_inplace_method:+            inplace_method = '__i' + method[2:]+            if typ.type.has_readable_member(inplace_method):+                return True, inplace_method+    return False, method+++def is_valid_inferred_type(typ: Type) -> bool:+    """Is an inferred type valid?++    Examples of invalid types include the None type or List[<uninhabited>].++    When not doing strict Optional checking, all types containing None are+    invalid.  When doing strict Optional checking, only None and types that are+    incompletely defined (i.e. contain UninhabitedType) are invalid.+    """+    if isinstance(typ, (NoneTyp, UninhabitedType)):+        # With strict Optional checking, we *may* eventually infer NoneTyp when+        # the initializer is None, but we only do that if we can't infer a+        # specific Optional type.  This resolution happens in+        # leave_partial_types when we pop a partial types scope.+        return False+    return is_valid_inferred_type_component(typ)+++def is_valid_inferred_type_component(typ: Type) -> bool:+    """Is this part of a type a valid inferred type?++    In strict Optional mode this excludes bare None types, as otherwise every+    type containing None would be invalid.+    """+    if is_same_type(typ, UninhabitedType()):+        return False+    elif isinstance(typ, Instance):+        for arg in typ.args:+            if not is_valid_inferred_type_component(arg):+                return False+    elif isinstance(typ, TupleType):+        for item in typ.items:+            if not is_valid_inferred_type_component(item):+                return False+    return True+++def is_node_static(node: Optional[Node]) -> Optional[bool]:+    """Find out if a node describes a static function method."""++    if isinstance(node, FuncDef):+        return node.is_static++    if isinstance(node, Var):+        return node.is_staticmethod++    return None+++class CheckerScope:+    # We keep two stacks combined, to maintain the relative order+    stack = None  # type: List[Union[TypeInfo, FuncItem, MypyFile]]++    def __init__(self, module: MypyFile) -> None:+        self.stack = [module]++    def top_function(self) -> Optional[FuncItem]:+        for e in reversed(self.stack):+            if isinstance(e, FuncItem):+                return e+        return None++    def top_non_lambda_function(self) -> Optional[FuncItem]:+        for e in reversed(self.stack):+            if isinstance(e, FuncItem) and not isinstance(e, LambdaExpr):+                return e+        return None++    def active_class(self) -> Optional[TypeInfo]:+        if isinstance(self.stack[-1], TypeInfo):+            return self.stack[-1]+        return None++    def enclosing_class(self) -> Optional[TypeInfo]:+        top = self.top_function()+        assert top, "This method must be called from inside a function"+        index = self.stack.index(top)+        assert index, "CheckerScope stack must always start with a module"+        enclosing = self.stack[index - 1]+        if isinstance(enclosing, TypeInfo):+            return enclosing+        return None++    def active_self_type(self) -> Optional[Union[Instance, TupleType]]:+        info = self.active_class()+        if info:+            return fill_typevars(info)+        return None++    @contextmanager+    def push_function(self, item: FuncItem) -> Iterator[None]:+        self.stack.append(item)+        yield+        self.stack.pop()++    @contextmanager+    def push_class(self, info: TypeInfo) -> Iterator[None]:+        self.stack.append(info)+        yield+        self.stack.pop()+++@contextmanager+def nothing() -> Iterator[None]:+    yield+++def is_typed_callable(c: Optional[Type]) -> bool:+    if not c or not isinstance(c, CallableType):+        return False+    return not all(isinstance(t, AnyType) and t.type_of_any == TypeOfAny.unannotated+                   for t in c.arg_types + [c.ret_type])+++def is_untyped_decorator(typ: Optional[Type]) -> bool:+    if not typ or not isinstance(typ, CallableType):+        return True+    return typ.implicit+++def is_static(func: Union[FuncBase, Decorator]) -> bool:+    if isinstance(func, Decorator):+        return is_static(func.func)+    elif isinstance(func, FuncBase):+        return func.is_static+    assert False, "Unexpected func type: {}".format(type(func))
+ test/files/mypy2.py view
@@ -0,0 +1,3800 @@+"""The semantic analyzer passes 1 and 2.++Bind names to definitions and do various other simple consistency+checks. For example, consider this program:++  x = 1+  y = x++Here semantic analysis would detect that the assignment 'x = 1'+defines a new variable, the type of which is to be inferred (in a+later pass; type inference or type checking is not part of semantic+analysis).  Also, it would bind both references to 'x' to the same+module-level variable (Var) node.  The second assignment would also+be analyzed, and the type of 'y' marked as being inferred.++Semantic analysis is the first analysis pass after parsing, and it is+subdivided into three passes:++ * SemanticAnalyzerPass1 is defined in mypy.semanal_pass1.++ * SemanticAnalyzerPass2 is the second pass.  It does the bulk of the work.+   It assumes that dependent modules have been semantically analyzed,+   up to the second pass, unless there is a import cycle.++ * SemanticAnalyzerPass3 is the third pass. It's in mypy.semanal_pass3.++Semantic analysis of types is implemented in module mypy.typeanal.++TODO: Check if the third pass slows down type checking significantly.+  We could probably get rid of it -- for example, we could collect all+  analyzed types in a collection and check them without having to+  traverse the entire AST.+"""++from contextlib import contextmanager++from typing import (+    List, Dict, Set, Tuple, cast, TypeVar, Union, Optional, Callable, Iterator, Iterable,+)++from mypy.nodes import (+    MypyFile, TypeInfo, Node, AssignmentStmt, FuncDef, OverloadedFuncDef,+    ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, Expression, Lvalue,+    ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr,+    IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt,+    RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt,+    ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt,+    GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr,+    SliceExpr, CastExpr, RevealExpr, TypeApplication, Context, SymbolTable,+    SymbolTableNode, TVAR, ListComprehension, GeneratorExpr,+    LambdaExpr, MDEF, Decorator, SetExpr, TypeVarExpr,+    StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr,+    ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, type_aliases,+    YieldFromExpr, NamedTupleExpr, NonlocalDecl, SymbolNode,+    SetComprehension, DictionaryComprehension, TypeAlias, TypeAliasExpr,+    YieldExpr, ExecStmt, BackquoteExpr, ImportBase, AwaitExpr,+    IntExpr, FloatExpr, UnicodeExpr, TempNode, ImportedName,+    COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED, LITERAL_YES, nongen_builtins,+    get_member_expr_fullname, REVEAL_TYPE, REVEAL_LOCALS+)+from mypy.literals import literal+from mypy.tvar_scope import TypeVarScope+from mypy.typevars import fill_typevars+from mypy.visitor import NodeVisitor+from mypy.traverser import TraverserVisitor+from mypy.errors import Errors, report_internal_error+from mypy.messages import CANNOT_ASSIGN_TO_TYPE, MessageBuilder+from mypy.types import (+    FunctionLike, UnboundType, TypeVarDef, TupleType, UnionType, StarType, function_type,+    CallableType, Overloaded, Instance, Type, AnyType,+    TypeTranslator, TypeOfAny+)+from mypy.nodes import implicit_module_attrs+from mypy.typeanal import (+    TypeAnalyser, analyze_type_alias, no_subscript_builtin_alias,+    TypeVariableQuery, TypeVarList, remove_dups, has_any_from_unimported_type,+    check_for_explicit_any+)+from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError+from mypy.sametypes import is_same_type+from mypy.options import Options+from mypy import experiments+from mypy.plugin import Plugin, ClassDefContext, SemanticAnalyzerPluginInterface+from mypy.util import get_prefix, correct_relative_import+from mypy.semanal_shared import SemanticAnalyzerInterface, set_callable_name+from mypy.scope import Scope+from mypy.semanal_namedtuple import NamedTupleAnalyzer, NAMEDTUPLE_PROHIBITED_NAMES+from mypy.semanal_typeddict import TypedDictAnalyzer+from mypy.semanal_enum import EnumCallAnalyzer+from mypy.semanal_newtype import NewTypeAnalyzer+from mypy.typestate import TypeState+++T = TypeVar('T')+++# Inferred truth value of an expression.+ALWAYS_TRUE = 1+MYPY_TRUE = 2  # True in mypy, False at runtime+ALWAYS_FALSE = 3+MYPY_FALSE = 4  # False in mypy, True at runtime+TRUTH_VALUE_UNKNOWN = 5++inverted_truth_mapping = {+    ALWAYS_TRUE: ALWAYS_FALSE,+    ALWAYS_FALSE: ALWAYS_TRUE,+    TRUTH_VALUE_UNKNOWN: TRUTH_VALUE_UNKNOWN,+    MYPY_TRUE: MYPY_FALSE,+    MYPY_FALSE: MYPY_TRUE,+}++# Map from obsolete name to the current spelling.+obsolete_name_mapping = {+    'typing.Function': 'typing.Callable',+    'typing.typevar': 'typing.TypeVar',+}++# Hard coded type promotions (shared between all Python versions).+# These add extra ad-hoc edges to the subtyping relation. For example,+# int is considered a subtype of float, even though there is no+# subclass relationship.+TYPE_PROMOTIONS = {+    'builtins.int': 'builtins.float',+    'builtins.float': 'builtins.complex',+}++# Hard coded type promotions for Python 3.+#+# Note that the bytearray -> bytes promotion is a little unsafe+# as some functions only accept bytes objects. Here convenience+# trumps safety.+TYPE_PROMOTIONS_PYTHON3 = TYPE_PROMOTIONS.copy()+TYPE_PROMOTIONS_PYTHON3.update({+    'builtins.bytearray': 'builtins.bytes',+})++# Hard coded type promotions for Python 2.+#+# These promotions are unsafe, but we are doing them anyway+# for convenience and also for Python 3 compatibility+# (bytearray -> str).+TYPE_PROMOTIONS_PYTHON2 = TYPE_PROMOTIONS.copy()+TYPE_PROMOTIONS_PYTHON2.update({+    'builtins.str': 'builtins.unicode',+    'builtins.bytearray': 'builtins.str',+})++# When analyzing a function, should we analyze the whole function in one go, or+# should we only perform one phase of the analysis? The latter is used for+# nested functions. In the first phase we add the function to the symbol table+# but don't process body. In the second phase we process function body. This+# way we can have mutually recursive nested functions.+FUNCTION_BOTH_PHASES = 0  # Everything in one go+FUNCTION_FIRST_PHASE_POSTPONE_SECOND = 1  # Add to symbol table but postpone body+FUNCTION_SECOND_PHASE = 2  # Only analyze body++# Map from the full name of a missing definition to the test fixture (under+# test-data/unit/fixtures/) that provides the definition. This is used for+# generating better error messages when running mypy tests only.+SUGGESTED_TEST_FIXTURES = {+    'builtins.list': 'list.pyi',+    'builtins.dict': 'dict.pyi',+    'builtins.set': 'set.pyi',+    'builtins.bool': 'bool.pyi',+    'builtins.Exception': 'exception.pyi',+    'builtins.BaseException': 'exception.pyi',+    'builtins.isinstance': 'isinstancelist.pyi',+    'builtins.property': 'property.pyi',+    'builtins.classmethod': 'classmethod.pyi',+}+++class SemanticAnalyzerPass2(NodeVisitor[None],+                            SemanticAnalyzerInterface,+                            SemanticAnalyzerPluginInterface):+    """Semantically analyze parsed mypy files.++    The analyzer binds names and does various consistency checks for a+    parse tree. Note that type checking is performed as a separate+    pass.++    This is the second phase of semantic analysis.+    """++    # Module name space+    modules = None  # type: Dict[str, MypyFile]+    # Global name space for current module+    globals = None  # type: SymbolTable+    # Names declared using "global" (separate set for each scope)+    global_decls = None  # type: List[Set[str]]+    # Names declated using "nonlocal" (separate set for each scope)+    nonlocal_decls = None  # type: List[Set[str]]+    # Local names of function scopes; None for non-function scopes.+    locals = None  # type: List[Optional[SymbolTable]]+    # Nested block depths of scopes+    block_depth = None  # type: List[int]+    # TypeInfo of directly enclosing class (or None)+    type = None  # type: Optional[TypeInfo]+    # Stack of outer classes (the second tuple item contains tvars).+    type_stack = None  # type: List[Optional[TypeInfo]]+    # Type variables bound by the current scope, be it class or function+    tvar_scope = None  # type: TypeVarScope+    # Per-module options+    options = None  # type: Options++    # Stack of functions being analyzed+    function_stack = None  # type: List[FuncItem]++    # Status of postponing analysis of nested function bodies. By using this we+    # can have mutually recursive nested functions. Values are FUNCTION_x+    # constants. Note that separate phasea are not used for methods.+    postpone_nested_functions_stack = None  # type: List[int]+    # Postponed functions collected if+    # postpone_nested_functions_stack[-1] == FUNCTION_FIRST_PHASE_POSTPONE_SECOND.+    postponed_functions_stack = None  # type: List[List[Node]]++    loop_depth = 0         # Depth of breakable loops+    cur_mod_id = ''        # Current module id (or None) (phase 2)+    is_stub_file = False   # Are we analyzing a stub file?+    _is_typeshed_stub_file = False  # Are we analyzing a typeshed stub file?+    imports = None  # type: Set[str]  # Imported modules (during phase 2 analysis)+    errors = None  # type: Errors     # Keeps track of generated errors+    plugin = None  # type: Plugin     # Mypy plugin for special casing of library features++    def __init__(self,+                 modules: Dict[str, MypyFile],+                 missing_modules: Set[str],+                 errors: Errors,+                 plugin: Plugin) -> None:+        """Construct semantic analyzer.++        Use lib_path to search for modules, and report analysis errors+        using the Errors instance.+        """+        self.locals = [None]+        self.imports = set()+        self.type = None+        self.type_stack = []+        self.tvar_scope = TypeVarScope()+        self.function_stack = []+        self.block_depth = [0]+        self.loop_depth = 0+        self.errors = errors+        self.modules = modules+        self.msg = MessageBuilder(errors, modules)+        self.missing_modules = missing_modules+        self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES]+        self.postponed_functions_stack = []+        self.all_exports = set()  # type: Set[str]+        self.plugin = plugin+        # If True, process function definitions. If False, don't. This is used+        # for processing module top levels in fine-grained incremental mode.+        self.recurse_into_functions = True+        self.scope = Scope()++    # mypyc doesn't properly handle implementing an abstractproperty+    # with a regular attribute so we make it a property+    @property+    def is_typeshed_stub_file(self) -> bool:+        return self._is_typeshed_stub_file++    def visit_file(self, file_node: MypyFile, fnam: str, options: Options,+                   patches: List[Tuple[int, Callable[[], None]]]) -> None:+        """Run semantic analysis phase 2 over a file.++        Add (priority, callback) pairs by mutating the 'patches' list argument. They+        will be called after all semantic analysis phases but before type checking,+        lowest priority values first.+        """+        self.recurse_into_functions = True+        self.options = options+        self.errors.set_file(fnam, file_node.fullname(), scope=self.scope)+        self.cur_mod_node = file_node+        self.cur_mod_id = file_node.fullname()+        self.is_stub_file = fnam.lower().endswith('.pyi')+        self._is_typeshed_stub_file = self.errors.is_typeshed_file(file_node.path)+        self.globals = file_node.names+        self.patches = patches+        self.named_tuple_analyzer = NamedTupleAnalyzer(options, self)+        self.typed_dict_analyzer = TypedDictAnalyzer(options, self, self.msg)+        self.enum_call_analyzer = EnumCallAnalyzer(options, self)+        self.newtype_analyzer = NewTypeAnalyzer(options, self, self.msg)++        with experiments.strict_optional_set(options.strict_optional):+            if 'builtins' in self.modules:+                self.globals['__builtins__'] = SymbolTableNode(MODULE_REF,+                                                               self.modules['builtins'])++            for name in implicit_module_attrs:+                v = self.globals[name].node+                if isinstance(v, Var):+                    assert v.type is not None, "Type of implicit attribute not set"+                    v.type = self.anal_type(v.type)+                    v.is_ready = True++            defs = file_node.defs+            self.scope.enter_file(file_node.fullname())+            for d in defs:+                self.accept(d)+            self.scope.leave()++            if self.cur_mod_id == 'builtins':+                remove_imported_names_from_symtable(self.globals, 'builtins')+                for alias_name in type_aliases:+                    self.globals.pop(alias_name.split('.')[-1], None)++            if '__all__' in self.globals:+                for name, g in self.globals.items():+                    if name not in self.all_exports:+                        g.module_public = False++            del self.options+            del self.patches+            del self.cur_mod_node+            del self.globals++    def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef],+                        patches: List[Tuple[int, Callable[[], None]]]) -> None:+        """Refresh a stale target in fine-grained incremental mode."""+        self.patches = patches+        if isinstance(node, MypyFile):+            self.refresh_top_level(node)+        else:+            self.recurse_into_functions = True+            self.accept(node)+        del self.patches++    def refresh_top_level(self, file_node: MypyFile) -> None:+        """Reanalyze a stale module top-level in fine-grained incremental mode."""+        self.recurse_into_functions = False+        for d in file_node.defs:+            self.accept(d)++    @contextmanager+    def file_context(self, file_node: MypyFile, fnam: str, options: Options,+                     active_type: Optional[TypeInfo],+                     scope: Optional[Scope] = None) -> Iterator[None]:+        # TODO: Use this above in visit_file+        scope = scope or self.scope+        self.options = options+        self.errors.set_file(fnam, file_node.fullname(), scope=scope)+        self.cur_mod_node = file_node+        self.cur_mod_id = file_node.fullname()+        scope.enter_file(self.cur_mod_id)+        self.is_stub_file = fnam.lower().endswith('.pyi')+        self._is_typeshed_stub_file = self.errors.is_typeshed_file(file_node.path)+        self.globals = file_node.names+        self.tvar_scope = TypeVarScope()+        if active_type:+            scope.enter_class(active_type)+            self.enter_class(active_type.defn.info)+            for tvar in active_type.defn.type_vars:+                self.tvar_scope.bind_existing(tvar)++        yield++        if active_type:+            scope.leave()+            self.leave_class()+            self.type = None+        scope.leave()+        del self.options++    def visit_func_def(self, defn: FuncDef) -> None:+        if not self.recurse_into_functions:+            return+        with self.scope.function_scope(defn):+            self._visit_func_def(defn)++    def _visit_func_def(self, defn: FuncDef) -> None:+        phase_info = self.postpone_nested_functions_stack[-1]+        if phase_info != FUNCTION_SECOND_PHASE:+            self.function_stack.append(defn)+            # First phase of analysis for function.+            if not defn._fullname:+                defn._fullname = self.qualified_name(defn.name())+            if defn.type:+                assert isinstance(defn.type, CallableType)+                self.update_function_type_variables(defn.type, defn)+            self.function_stack.pop()++            defn.is_conditional = self.block_depth[-1] > 0++            # TODO(jukka): Figure out how to share the various cases. It doesn't+            #   make sense to have (almost) duplicate code (here and elsewhere) for+            #   3 cases: module-level, class-level and local names. Maybe implement+            #   a common stack of namespaces. As the 3 kinds of namespaces have+            #   different semantics, this wouldn't always work, but it might still+            #   be a win.+            if self.is_class_scope():+                # Method definition+                assert self.type is not None, "Type not set at class scope"+                defn.info = self.type+                if not defn.is_decorated and not defn.is_overload:+                    if (defn.name() in self.type.names and+                            self.type.names[defn.name()].node != defn):+                        # Redefinition. Conditional redefinition is okay.+                        n = self.type.names[defn.name()].node+                        if not self.set_original_def(n, defn):+                            self.name_already_defined(defn.name(), defn,+                                                      self.type.names[defn.name()])+                    self.type.names[defn.name()] = SymbolTableNode(MDEF, defn)+                self.prepare_method_signature(defn, self.type)+            elif self.is_func_scope():+                # Nested function+                assert self.locals[-1] is not None, "No locals at function scope"+                if not defn.is_decorated and not defn.is_overload:+                    if defn.name() in self.locals[-1]:+                        # Redefinition. Conditional redefinition is okay.+                        n = self.locals[-1][defn.name()].node+                        if not self.set_original_def(n, defn):+                            self.name_already_defined(defn.name(), defn,+                                                      self.locals[-1][defn.name()])+                    else:+                        self.add_local(defn, defn)+            else:+                # Top-level function+                if not defn.is_decorated and not defn.is_overload:+                    symbol = self.globals[defn.name()]+                    if isinstance(symbol.node, FuncDef) and symbol.node != defn:+                        # This is redefinition. Conditional redefinition is okay.+                        if not self.set_original_def(symbol.node, defn):+                            # Report error.+                            self.check_no_global(defn.name(), defn, True)++            # Analyze function signature and initializers in the first phase+            # (at least this mirrors what happens at runtime).+            with self.tvar_scope_frame(self.tvar_scope.method_frame()):+                if defn.type:+                    self.check_classvar_in_signature(defn.type)+                    assert isinstance(defn.type, CallableType)+                    # Signature must be analyzed in the surrounding scope so that+                    # class-level imported names and type variables are in scope.+                    analyzer = self.type_analyzer()+                    defn.type = analyzer.visit_callable_type(defn.type, nested=False)+                    self.add_type_alias_deps(analyzer.aliases_used)+                    self.check_function_signature(defn)+                    if isinstance(defn, FuncDef):+                        assert isinstance(defn.type, CallableType)+                        defn.type = set_callable_name(defn.type, defn)+                for arg in defn.arguments:+                    if arg.initializer:+                        arg.initializer.accept(self)++            if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND:+                # Postpone this function (for the second phase).+                self.postponed_functions_stack[-1].append(defn)+                return+        if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND:+            # Second phase of analysis for function.+            self.analyze_function(defn)+            if defn.is_coroutine and isinstance(defn.type, CallableType):+                if defn.is_async_generator:+                    # Async generator types are handled elsewhere+                    pass+                else:+                    # A coroutine defined as `async def foo(...) -> T: ...`+                    # has external return type `Coroutine[Any, Any, T]`.+                    any_type = AnyType(TypeOfAny.special_form)+                    ret_type = self.named_type_or_none('typing.Coroutine',+                        [any_type, any_type, defn.type.ret_type])+                    assert ret_type is not None, "Internal error: typing.Coroutine not found"+                    defn.type = defn.type.copy_modified(ret_type=ret_type)++    def prepare_method_signature(self, func: FuncDef, info: TypeInfo) -> None:+        """Check basic signature validity and tweak annotation of self/cls argument."""+        # Only non-static methods are special.+        functype = func.type+        if not func.is_static:+            if not func.arguments:+                self.fail('Method must have at least one argument', func)+            elif isinstance(functype, CallableType):+                self_type = functype.arg_types[0]+                if isinstance(self_type, AnyType):+                    if func.is_class or func.name() in ('__new__', '__init_subclass__'):+                        leading_type = self.class_type(info)+                    else:+                        leading_type = fill_typevars(info)+                    func.type = replace_implicit_first_type(functype, leading_type)++    def set_original_def(self, previous: Optional[Node], new: FuncDef) -> bool:+        """If 'new' conditionally redefine 'previous', set 'previous' as original++        We reject straight redefinitions of functions, as they are usually+        a programming error. For example:++        . def f(): ...+        . def f(): ...  # Error: 'f' redefined+        """+        if isinstance(previous, (FuncDef, Var, Decorator)) and new.is_conditional:+            new.original_def = previous+            return True+        else:+            return False++    def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) -> None:+        """Make any type variables in the signature of defn explicit.++        Update the signature of defn to contain type variable definitions+        if defn is generic.+        """+        with self.tvar_scope_frame(self.tvar_scope.method_frame()):+            a = self.type_analyzer()+            fun_type.variables = a.bind_function_type_variables(fun_type, defn)++    def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:+        if not self.recurse_into_functions:+            return+        # NB: Since _visit_overloaded_func_def will call accept on the+        # underlying FuncDefs, the function might get entered twice.+        # This is fine, though, because only the outermost function is+        # used to compute targets.+        with self.scope.function_scope(defn):+            self._visit_overloaded_func_def(defn)++    def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:+        # OverloadedFuncDef refers to any legitimate situation where you have+        # more than one declaration for the same function in a row.  This occurs+        # with a @property with a setter or a deleter, and for a classic+        # @overload.++        # Decide whether to analyze this as a property or an overload.  If an+        # overload, and we're outside a stub, find the impl and set it.  Remove+        # the impl from the item list, it's special.+        types = []  # type: List[CallableType]+        non_overload_indexes = []++        # See if the first item is a property (and not an overload)+        first_item = defn.items[0]+        first_item.is_overload = True+        first_item.accept(self)++        defn._fullname = self.qualified_name(defn.name())++        if isinstance(first_item, Decorator) and first_item.func.is_property:+            first_item.func.is_overload = True+            self.analyze_property_with_multi_part_definition(defn)+            typ = function_type(first_item.func, self.builtin_type('builtins.function'))+            assert isinstance(typ, CallableType)+            types = [typ]+        else:+            for i, item in enumerate(defn.items):+                if i != 0:+                    # The first item was already visited+                    item.is_overload = True+                    item.accept(self)+                # TODO: support decorated overloaded functions properly+                if isinstance(item, Decorator):+                    callable = function_type(item.func, self.builtin_type('builtins.function'))+                    assert isinstance(callable, CallableType)+                    if not any(refers_to_fullname(dec, 'typing.overload')+                               for dec in item.decorators):+                        if i == len(defn.items) - 1 and not self.is_stub_file:+                            # Last item outside a stub is impl+                            defn.impl = item+                        else:+                            # Oops it wasn't an overload after all. A clear error+                            # will vary based on where in the list it is, record+                            # that.+                            non_overload_indexes.append(i)+                    else:+                        item.func.is_overload = True+                        types.append(callable)+                elif isinstance(item, FuncDef):+                    if i == len(defn.items) - 1 and not self.is_stub_file:+                        defn.impl = item+                    else:+                        non_overload_indexes.append(i)+            if non_overload_indexes:+                if types:+                    # Some of them were overloads, but not all.+                    for idx in non_overload_indexes:+                        if self.is_stub_file:+                            self.fail("An implementation for an overloaded function "+                                      "is not allowed in a stub file", defn.items[idx])+                        else:+                            self.fail("The implementation for an overloaded function "+                                      "must come last", defn.items[idx])+                else:+                    for idx in non_overload_indexes[1:]:+                        self.name_already_defined(defn.name(), defn.items[idx], first_item)+                    if defn.impl:+                        self.name_already_defined(defn.name(), defn.impl, first_item)+                # Remove the non-overloads+                for idx in reversed(non_overload_indexes):+                    del defn.items[idx]+            # If we found an implementation, remove it from the overloads to+            # consider.+            if defn.impl is not None:+                assert defn.impl is defn.items[-1]+                defn.items = defn.items[:-1]+            elif not self.is_stub_file and not non_overload_indexes:+                if not (self.type and not self.is_func_scope() and self.type.is_protocol):+                    self.fail(+                        "An overloaded function outside a stub file must have an implementation",+                        defn)+                else:+                    for item in defn.items:+                        if isinstance(item, Decorator):+                            item.func.is_abstract = True+                        else:+                            item.is_abstract = True++        if types:+            defn.type = Overloaded(types)+            defn.type.line = defn.line++        if not defn.items:+            # It was not any kind of overload def after all. We've visited the+            # redefinitions already.+            return++        # We know this is an overload def -- let's handle classmethod and staticmethod+        class_status = []+        static_status = []+        for item in defn.items:+            if isinstance(item, Decorator):+                inner = item.func+            elif isinstance(item, FuncDef):+                inner = item+            else:+                assert False, "The 'item' variable is an unexpected type: {}".format(type(item))+            class_status.append(inner.is_class)+            static_status.append(inner.is_static)++        if defn.impl is not None:+            if isinstance(defn.impl, Decorator):+                inner = defn.impl.func+            elif isinstance(defn.impl, FuncDef):+                inner = defn.impl+            else:+                assert False, "Unexpected impl type: {}".format(type(defn.impl))+            class_status.append(inner.is_class)+            static_status.append(inner.is_static)++        if len(set(class_status)) != 1:+            self.msg.overload_inconsistently_applies_decorator('classmethod', defn)+        elif len(set(static_status)) != 1:+            self.msg.overload_inconsistently_applies_decorator('staticmethod', defn)+        else:+            defn.is_class = class_status[0]+            defn.is_static = static_status[0]++        if self.type and not self.is_func_scope():+            self.type.names[defn.name()] = SymbolTableNode(MDEF, defn)+            defn.info = self.type+        elif self.is_func_scope():+            self.add_local(defn, defn)++    def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) -> None:+        """Analyze a property defined using multiple methods (e.g., using @x.setter).++        Assume that the first method (@property) has already been analyzed.+        """+        defn.is_property = True+        items = defn.items+        first_item = cast(Decorator, defn.items[0])+        for item in items[1:]:+            if isinstance(item, Decorator) and len(item.decorators) == 1:+                node = item.decorators[0]+                if isinstance(node, MemberExpr):+                    if node.name == 'setter':+                        # The first item represents the entire property.+                        first_item.var.is_settable_property = True+                        # Get abstractness from the original definition.+                        item.func.is_abstract = first_item.func.is_abstract+            else:+                self.fail("Decorated property not supported", item)+            if isinstance(item, Decorator):+                item.func.accept(self)++    def analyze_function(self, defn: FuncItem) -> None:+        is_method = self.is_class_scope()+        with self.tvar_scope_frame(self.tvar_scope.method_frame()):+            # Bind the type variables again to visit the body.+            if defn.type:+                a = self.type_analyzer()+                a.bind_function_type_variables(cast(CallableType, defn.type), defn)+            self.function_stack.append(defn)+            self.enter()+            for arg in defn.arguments:+                self.add_local(arg.variable, defn)++            # The first argument of a non-static, non-class method is like 'self'+            # (though the name could be different), having the enclosing class's+            # instance type.+            if is_method and not defn.is_static and not defn.is_class and defn.arguments:+                defn.arguments[0].variable.is_self = True++            # First analyze body of the function but ignore nested functions.+            self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND)+            self.postponed_functions_stack.append([])+            defn.body.accept(self)++            # Analyze nested functions (if any) as a second phase.+            self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE+            for postponed in self.postponed_functions_stack[-1]:+                postponed.accept(self)+            self.postpone_nested_functions_stack.pop()+            self.postponed_functions_stack.pop()++            self.leave()+            self.function_stack.pop()++    def check_classvar_in_signature(self, typ: Type) -> None:+        if isinstance(typ, Overloaded):+            for t in typ.items():  # type: Type+                self.check_classvar_in_signature(t)+            return+        if not isinstance(typ, CallableType):+            return+        for t in typ.arg_types + [typ.ret_type]:+            if self.is_classvar(t):+                self.fail_invalid_classvar(t)+                # Show only one error per signature+                break++    def check_function_signature(self, fdef: FuncItem) -> None:+        sig = fdef.type+        assert isinstance(sig, CallableType)+        if len(sig.arg_types) < len(fdef.arguments):+            self.fail('Type signature has too few arguments', fdef)+            # Add dummy Any arguments to prevent crashes later.+            num_extra_anys = len(fdef.arguments) - len(sig.arg_types)+            extra_anys = [AnyType(TypeOfAny.from_error)] * num_extra_anys+            sig.arg_types.extend(extra_anys)+        elif len(sig.arg_types) > len(fdef.arguments):+            self.fail('Type signature has too many arguments', fdef, blocker=True)++    def visit_class_def(self, defn: ClassDef) -> None:+        with self.scope.class_scope(defn.info):+            with self.analyze_class_body(defn) as should_continue:+                if should_continue:+                    # Analyze class body.+                    defn.defs.accept(self)++    @contextmanager+    def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]:+        with self.tvar_scope_frame(self.tvar_scope.class_frame()):+            is_protocol = self.detect_protocol_base(defn)+            self.update_metaclass(defn)+            self.clean_up_bases_and_infer_type_variables(defn)+            self.analyze_class_keywords(defn)+            if self.typed_dict_analyzer.analyze_typeddict_classdef(defn):+                yield False+                return+            named_tuple_info = self.named_tuple_analyzer.analyze_namedtuple_classdef(defn)+            if named_tuple_info is not None:+                # Temporarily clear the names dict so we don't get errors about duplicate names+                # that were already set in build_namedtuple_typeinfo.+                nt_names = named_tuple_info.names+                named_tuple_info.names = SymbolTable()+                # This is needed for the cls argument to classmethods to get bound correctly.+                named_tuple_info.names['__init__'] = nt_names['__init__']++                self.enter_class(named_tuple_info)++                yield True++                self.leave_class()++                # make sure we didn't use illegal names, then reset the names in the typeinfo+                for prohibited in NAMEDTUPLE_PROHIBITED_NAMES:+                    if prohibited in named_tuple_info.names:+                        if nt_names.get(prohibited) is named_tuple_info.names[prohibited]:+                            continue+                        ctx = named_tuple_info.names[prohibited].node+                        assert ctx is not None+                        self.fail('Cannot overwrite NamedTuple attribute "{}"'.format(prohibited),+                                  ctx)++                # Restore the names in the original symbol table. This ensures that the symbol+                # table contains the field objects created by build_namedtuple_typeinfo. Exclude+                # __doc__, which can legally be overwritten by the class.+                named_tuple_info.names.update({+                    key: value for key, value in nt_names.items()+                    if key not in named_tuple_info.names or key != '__doc__'+                })+            else:+                self.setup_class_def_analysis(defn)+                self.analyze_base_classes(defn)+                self.analyze_metaclass(defn)+                defn.info.is_protocol = is_protocol+                defn.info.runtime_protocol = False+                for decorator in defn.decorators:+                    self.analyze_class_decorator(defn, decorator)+                self.enter_class(defn.info)+                yield True+                self.calculate_abstract_status(defn.info)+                self.setup_type_promotion(defn)+                self.apply_class_plugin_hooks(defn)+                self.leave_class()++    def apply_class_plugin_hooks(self, defn: ClassDef) -> None:+        """Apply a plugin hook that may infer a more precise definition for a class."""+        def get_fullname(expr: Expression) -> Optional[str]:+            if isinstance(expr, CallExpr):+                return get_fullname(expr.callee)+            elif isinstance(expr, IndexExpr):+                return get_fullname(expr.base)+            elif isinstance(expr, RefExpr):+                if expr.fullname:+                    return expr.fullname+                # If we don't have a fullname look it up. This happens because base classes are+                # analyzed in a different manner (see exprtotype.py) and therefore those AST+                # nodes will not have full names.+                sym = self.lookup_type_node(expr)+                if sym:+                    return sym.fullname+            return None++        for decorator in defn.decorators:+            decorator_name = get_fullname(decorator)+            if decorator_name:+                hook = self.plugin.get_class_decorator_hook(decorator_name)+                if hook:+                    hook(ClassDefContext(defn, decorator, self))++        if defn.metaclass:+            metaclass_name = get_fullname(defn.metaclass)+            if metaclass_name:+                hook = self.plugin.get_metaclass_hook(metaclass_name)+                if hook:+                    hook(ClassDefContext(defn, defn.metaclass, self))++        for base_expr in defn.base_type_exprs:+            base_name = get_fullname(base_expr)+            if base_name:+                hook = self.plugin.get_base_class_hook(base_name)+                if hook:+                    hook(ClassDefContext(defn, base_expr, self))++    def analyze_class_keywords(self, defn: ClassDef) -> None:+        for value in defn.keywords.values():+            value.accept(self)++    def enter_class(self, info: TypeInfo) -> None:+        # Remember previous active class+        self.type_stack.append(self.type)+        self.locals.append(None)  # Add class scope+        self.block_depth.append(-1)  # The class body increments this to 0+        self.postpone_nested_functions_stack.append(FUNCTION_BOTH_PHASES)+        self.type = info++    def leave_class(self) -> None:+        """ Restore analyzer state. """+        self.postpone_nested_functions_stack.pop()+        self.block_depth.pop()+        self.locals.pop()+        self.type = self.type_stack.pop()++    def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None:+        decorator.accept(self)+        if (isinstance(decorator, RefExpr) and+                decorator.fullname in ('typing.runtime', 'typing_extensions.runtime')):+            if defn.info.is_protocol:+                defn.info.runtime_protocol = True+            else:+                self.fail('@runtime can only be used with protocol classes', defn)++    def calculate_abstract_status(self, typ: TypeInfo) -> None:+        """Calculate abstract status of a class.++        Set is_abstract of the type to True if the type has an unimplemented+        abstract attribute.  Also compute a list of abstract attributes.+        """+        concrete = set()  # type: Set[str]+        abstract = []  # type: List[str]+        abstract_in_this_class = []  # type: List[str]+        for base in typ.mro:+            for name, symnode in base.names.items():+                node = symnode.node+                if isinstance(node, OverloadedFuncDef):+                    # Unwrap an overloaded function definition. We can just+                    # check arbitrarily the first overload item. If the+                    # different items have a different abstract status, there+                    # should be an error reported elsewhere.+                    func = node.items[0]  # type: Optional[Node]+                else:+                    func = node+                if isinstance(func, Decorator):+                    fdef = func.func+                    if fdef.is_abstract and name not in concrete:+                        typ.is_abstract = True+                        abstract.append(name)+                        if base is typ:+                            abstract_in_this_class.append(name)+                elif isinstance(node, Var):+                    if node.is_abstract_var and name not in concrete:+                        typ.is_abstract = True+                        abstract.append(name)+                        if base is typ:+                            abstract_in_this_class.append(name)+                concrete.add(name)+        # In stubs, abstract classes need to be explicitly marked because it is too+        # easy to accidentally leave a concrete class abstract by forgetting to+        # implement some methods.+        typ.abstract_attributes = sorted(abstract)+        if not self.is_stub_file:+            return+        if (typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta'):+            return+        if typ.is_protocol:+            return+        if abstract and not abstract_in_this_class:+            attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract))+            self.fail("Class {} has abstract attributes {}".format(typ.fullname(), attrs), typ)+            self.note("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass",+                      typ)++    def setup_type_promotion(self, defn: ClassDef) -> None:+        """Setup extra, ad-hoc subtyping relationships between classes (promotion).++        This includes things like 'int' being compatible with 'float'.+        """+        promote_target = None  # type: Optional[Type]+        for decorator in defn.decorators:+            if isinstance(decorator, CallExpr):+                analyzed = decorator.analyzed+                if isinstance(analyzed, PromoteExpr):+                    # _promote class decorator (undocumented feature).+                    promote_target = analyzed.type+        if not promote_target:+            promotions = (TYPE_PROMOTIONS_PYTHON3 if self.options.python_version[0] >= 3+                          else TYPE_PROMOTIONS_PYTHON2)+            if defn.fullname in promotions:+                promote_target = self.named_type_or_none(promotions[defn.fullname])+        defn.info._promote = promote_target++    def detect_protocol_base(self, defn: ClassDef) -> bool:+        for base_expr in defn.base_type_exprs:+            try:+                base = expr_to_unanalyzed_type(base_expr)+            except TypeTranslationError:+                continue  # This will be reported later+            if not isinstance(base, UnboundType):+                continue+            sym = self.lookup_qualified(base.name, base)+            if sym is None or sym.node is None:+                continue+            if sym.node.fullname() in ('typing.Protocol', 'typing_extensions.Protocol'):+                return True+        return False++    def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None:+        """Remove extra base classes such as Generic and infer type vars.++        For example, consider this class:++        . class Foo(Bar, Generic[T]): ...++        Now we will remove Generic[T] from bases of Foo and infer that the+        type variable 'T' is a type argument of Foo.++        Note that this is performed *before* semantic analysis.+        """+        removed = []  # type: List[int]+        declared_tvars = []  # type: TypeVarList+        for i, base_expr in enumerate(defn.base_type_exprs):+            self.analyze_type_expr(base_expr)++            try:+                base = expr_to_unanalyzed_type(base_expr)+            except TypeTranslationError:+                # This error will be caught later.+                continue+            tvars = self.analyze_typevar_declaration(base)+            if tvars is not None:+                if declared_tvars:+                    self.fail('Only single Generic[...] or Protocol[...] can be in bases', defn)+                removed.append(i)+                declared_tvars.extend(tvars)+            if isinstance(base, UnboundType):+                sym = self.lookup_qualified(base.name, base)+                if sym is not None and sym.node is not None:+                    if (sym.node.fullname() in ('typing.Protocol',+                                                'typing_extensions.Protocol') and+                            i not in removed):+                        # also remove bare 'Protocol' bases+                        removed.append(i)++        all_tvars = self.get_all_bases_tvars(defn, removed)+        if declared_tvars:+            if len(remove_dups(declared_tvars)) < len(declared_tvars):+                self.fail("Duplicate type variables in Generic[...] or Protocol[...]", defn)+            declared_tvars = remove_dups(declared_tvars)+            if not set(all_tvars).issubset(set(declared_tvars)):+                self.fail("If Generic[...] or Protocol[...] is present"+                          " it should list all type variables", defn)+                # In case of error, Generic tvars will go first+                declared_tvars = remove_dups(declared_tvars + all_tvars)+        else:+            declared_tvars = all_tvars+        if declared_tvars:+            if defn.info:+                defn.info.type_vars = [name for name, _ in declared_tvars]+        for i in reversed(removed):+            defn.removed_base_type_exprs.append(defn.base_type_exprs[i])+            del defn.base_type_exprs[i]+        tvar_defs = []  # type: List[TypeVarDef]+        for name, tvar_expr in declared_tvars:+            tvar_def = self.tvar_scope.bind_new(name, tvar_expr)+            tvar_defs.append(tvar_def)+        defn.type_vars = tvar_defs++    def analyze_typevar_declaration(self, t: Type) -> Optional[TypeVarList]:+        if not isinstance(t, UnboundType):+            return None+        unbound = t+        sym = self.lookup_qualified(unbound.name, unbound)+        if sym is None or sym.node is None:+            return None+        if (sym.node.fullname() == 'typing.Generic' or+                sym.node.fullname() == 'typing.Protocol' and t.args or+                sym.node.fullname() == 'typing_extensions.Protocol' and t.args):+            tvars = []  # type: TypeVarList+            for arg in unbound.args:+                tvar = self.analyze_unbound_tvar(arg)+                if tvar:+                    tvars.append(tvar)+                else:+                    self.fail('Free type variable expected in %s[...]' %+                              sym.node.name(), t)+            return tvars+        return None++    def analyze_unbound_tvar(self, t: Type) -> Optional[Tuple[str, TypeVarExpr]]:+        if not isinstance(t, UnboundType):+            return None+        unbound = t+        sym = self.lookup_qualified(unbound.name, unbound)+        if sym is None or sym.kind != TVAR:+            return None+        elif sym.fullname and not self.tvar_scope.allow_binding(sym.fullname):+            # It's bound by our type variable scope+            return None+        else:+            assert isinstance(sym.node, TypeVarExpr)+            return unbound.name, sym.node++    def get_all_bases_tvars(self, defn: ClassDef, removed: List[int]) -> TypeVarList:+        tvars = []  # type: TypeVarList+        for i, base_expr in enumerate(defn.base_type_exprs):+            if i not in removed:+                try:+                    base = expr_to_unanalyzed_type(base_expr)+                except TypeTranslationError:+                    # This error will be caught later.+                    continue+                base_tvars = base.accept(TypeVariableQuery(self.lookup_qualified, self.tvar_scope))+                tvars.extend(base_tvars)+        return remove_dups(tvars)++    def setup_class_def_analysis(self, defn: ClassDef) -> None:+        """Prepare for the analysis of a class definition."""+        if not defn.info:+            defn.info = TypeInfo(SymbolTable(), defn, self.cur_mod_id)+            defn.info._fullname = defn.info.name()+        if self.is_func_scope() or self.type:+            kind = MDEF+            if self.is_nested_within_func_scope():+                kind = LDEF+            node = SymbolTableNode(kind, defn.info)+            self.add_symbol(defn.name, node, defn)+            if kind == LDEF:+                # We need to preserve local classes, let's store them+                # in globals under mangled unique names+                #+                # TODO: Putting local classes into globals breaks assumptions in fine-grained+                #     incremental mode and we should avoid it.+                if '@' not in defn.info._fullname:+                    local_name = defn.info._fullname + '@' + str(defn.line)+                    defn.info._fullname = self.cur_mod_id + '.' + local_name+                else:+                    # Preserve name from previous fine-grained incremental run.+                    local_name = defn.info._fullname+                defn.fullname = defn.info._fullname+                self.globals[local_name] = node++    def analyze_base_classes(self, defn: ClassDef) -> None:+        """Analyze and set up base classes.++        This computes several attributes on the corresponding TypeInfo defn.info+        related to the base classes: defn.info.bases, defn.info.mro, and+        miscellaneous others (at least tuple_type, fallback_to_any, and is_enum.)+        """++        base_types = []  # type: List[Instance]+        info = defn.info++        for base_expr in defn.base_type_exprs:+            try:+                base = self.expr_to_analyzed_type(base_expr)+            except TypeTranslationError:+                self.fail('Invalid base class', base_expr)+                info.fallback_to_any = True+                continue++            if isinstance(base, TupleType):+                if info.tuple_type:+                    self.fail("Class has two incompatible bases derived from tuple", defn)+                    defn.has_incompatible_baseclass = True+                info.tuple_type = base+                base_types.append(base.fallback)+                if isinstance(base_expr, CallExpr):+                    defn.analyzed = NamedTupleExpr(base.fallback.type)+                    defn.analyzed.line = defn.line+                    defn.analyzed.column = defn.column+            elif isinstance(base, Instance):+                if base.type.is_newtype:+                    self.fail("Cannot subclass NewType", defn)+                base_types.append(base)+            elif isinstance(base, AnyType):+                if self.options.disallow_subclassing_any:+                    if isinstance(base_expr, (NameExpr, MemberExpr)):+                        msg = "Class cannot subclass '{}' (has type 'Any')".format(base_expr.name)+                    else:+                        msg = "Class cannot subclass value of type 'Any'"+                    self.fail(msg, base_expr)+                info.fallback_to_any = True+            else:+                self.fail('Invalid base class', base_expr)+                info.fallback_to_any = True+            if self.options.disallow_any_unimported and has_any_from_unimported_type(base):+                if isinstance(base_expr, (NameExpr, MemberExpr)):+                    prefix = "Base type {}".format(base_expr.name)+                else:+                    prefix = "Base type"+                self.msg.unimported_type_becomes_any(prefix, base, base_expr)+            check_for_explicit_any(base, self.options, self.is_typeshed_stub_file, self.msg,+                                   context=base_expr)++        # Add 'object' as implicit base if there is no other base class.+        if (not base_types and defn.fullname != 'builtins.object'):+            base_types.append(self.object_type())++        info.bases = base_types++        # Calculate the MRO. It might be incomplete at this point if+        # the bases of defn include classes imported from other+        # modules in an import loop. We'll recompute it in SemanticAnalyzerPass3.+        if not self.verify_base_classes(defn):+            # Give it an MRO consisting of just the class itself and object.+            defn.info.mro = [defn.info, self.object_type().type]+            return+        calculate_class_mro(defn, self.fail_blocker)+        # If there are cyclic imports, we may be missing 'object' in+        # the MRO. Fix MRO if needed.+        if info.mro and info.mro[-1].fullname() != 'builtins.object':+            info.mro.append(self.object_type().type)++    def update_metaclass(self, defn: ClassDef) -> None:+        """Lookup for special metaclass declarations, and update defn fields accordingly.++        * __metaclass__ attribute in Python 2+        * six.with_metaclass(M, B1, B2, ...)+        * @six.add_metaclass(M)+        """++        # Look for "__metaclass__ = <metaclass>" in Python 2+        python2_meta_expr = None  # type: Optional[Expression]+        if self.options.python_version[0] == 2:+            for body_node in defn.defs.body:+                if isinstance(body_node, ClassDef) and body_node.name == "__metaclass__":+                    self.fail("Metaclasses defined as inner classes are not supported", body_node)+                    break+                elif isinstance(body_node, AssignmentStmt) and len(body_node.lvalues) == 1:+                    lvalue = body_node.lvalues[0]+                    if isinstance(lvalue, NameExpr) and lvalue.name == "__metaclass__":+                        python2_meta_expr = body_node.rvalue++        # Look for six.with_metaclass(M, B1, B2, ...)+        with_meta_expr = None  # type: Optional[Expression]+        if len(defn.base_type_exprs) == 1:+            base_expr = defn.base_type_exprs[0]+            if isinstance(base_expr, CallExpr) and isinstance(base_expr.callee, RefExpr):+                base_expr.callee.accept(self)+                if (base_expr.callee.fullname == 'six.with_metaclass'+                        and len(base_expr.args) >= 1+                        and all(kind == ARG_POS for kind in base_expr.arg_kinds)):+                    with_meta_expr = base_expr.args[0]+                    defn.base_type_exprs = base_expr.args[1:]++        # Look for @six.add_metaclass(M)+        add_meta_expr = None  # type: Optional[Expression]+        for dec_expr in defn.decorators:+            if isinstance(dec_expr, CallExpr) and isinstance(dec_expr.callee, RefExpr):+                dec_expr.callee.accept(self)+                if (dec_expr.callee.fullname == 'six.add_metaclass'+                    and len(dec_expr.args) == 1+                        and dec_expr.arg_kinds[0] == ARG_POS):+                    add_meta_expr = dec_expr.args[0]+                    break++        metas = {defn.metaclass, python2_meta_expr, with_meta_expr, add_meta_expr} - {None}+        if len(metas) == 0:+            return+        if len(metas) > 1:+            self.fail("Multiple metaclass definitions", defn)+            return+        defn.metaclass = metas.pop()++    def expr_to_analyzed_type(self, expr: Expression) -> Type:+        if isinstance(expr, CallExpr):+            expr.accept(self)+            info = self.named_tuple_analyzer.check_namedtuple(expr, None, self.is_func_scope())+            if info is None:+                # Some form of namedtuple is the only valid type that looks like a call+                # expression. This isn't a valid type.+                raise TypeTranslationError()+            assert info.tuple_type, "NamedTuple without tuple type"+            fallback = Instance(info, [])+            return TupleType(info.tuple_type.items, fallback=fallback)+        typ = expr_to_unanalyzed_type(expr)+        return self.anal_type(typ)++    def verify_base_classes(self, defn: ClassDef) -> bool:+        info = defn.info+        for base in info.bases:+            baseinfo = base.type+            if self.is_base_class(info, baseinfo):+                self.fail('Cycle in inheritance hierarchy', defn, blocker=True)+                # Clear bases to forcefully get rid of the cycle.+                info.bases = []+            if baseinfo.fullname() == 'builtins.bool':+                self.fail("'%s' is not a valid base class" %+                          baseinfo.name(), defn, blocker=True)+                return False+        dup = find_duplicate(info.direct_base_classes())+        if dup:+            self.fail('Duplicate base class "%s"' % dup.name(), defn, blocker=True)+            return False+        return True++    def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool:+        """Determine if t is a base class of s (but do not use mro)."""+        # Search the base class graph for t, starting from s.+        worklist = [s]+        visited = {s}+        while worklist:+            nxt = worklist.pop()+            if nxt == t:+                return True+            for base in nxt.bases:+                if base.type not in visited:+                    worklist.append(base.type)+                    visited.add(base.type)+        return False++    def analyze_metaclass(self, defn: ClassDef) -> None:+        if defn.metaclass:+            metaclass_name = None+            if isinstance(defn.metaclass, NameExpr):+                metaclass_name = defn.metaclass.name+            elif isinstance(defn.metaclass, MemberExpr):+                metaclass_name = get_member_expr_fullname(defn.metaclass)+            if metaclass_name is None:+                self.fail("Dynamic metaclass not supported for '%s'" % defn.name, defn.metaclass)+                return+            sym = self.lookup_qualified(metaclass_name, defn.metaclass)+            if sym is None:+                # Probably a name error - it is already handled elsewhere+                return+            if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType):+                # 'Any' metaclass -- just ignore it.+                #+                # TODO: A better approach would be to record this information+                #       and assume that the type object supports arbitrary+                #       attributes, similar to an 'Any' base class.+                return+            if not isinstance(sym.node, TypeInfo) or sym.node.tuple_type is not None:+                self.fail("Invalid metaclass '%s'" % metaclass_name, defn.metaclass)+                return+            if not sym.node.is_metaclass():+                self.fail("Metaclasses not inheriting from 'type' are not supported",+                          defn.metaclass)+                return+            inst = fill_typevars(sym.node)+            assert isinstance(inst, Instance)+            defn.info.declared_metaclass = inst+        defn.info.metaclass_type = defn.info.calculate_metaclass_type()+        if defn.info.metaclass_type is None:+            # Inconsistency may happen due to multiple baseclasses even in classes that+            # do not declare explicit metaclass, but it's harder to catch at this stage+            if defn.metaclass is not None:+                self.fail("Inconsistent metaclass structure for '%s'" % defn.name, defn)+        else:+            if defn.info.metaclass_type.type.has_base('enum.EnumMeta'):+                defn.info.is_enum = True+                if defn.type_vars:+                    self.fail("Enum class cannot be generic", defn)++    def object_type(self) -> Instance:+        return self.named_type('__builtins__.object')++    def str_type(self) -> Instance:+        return self.named_type('__builtins__.str')++    def class_type(self, info: TypeInfo) -> Type:+        # Construct a function type whose fallback is cls.+        from mypy import checkmember  # To avoid import cycle.+        leading_type = checkmember.type_object_type(info, self.builtin_type)+        if isinstance(leading_type, Overloaded):+            # Overloaded __init__ is too complex to handle.  Plus it's stubs only.+            return AnyType(TypeOfAny.special_form)+        else:+            return leading_type++    def named_type(self, qualified_name: str, args: Optional[List[Type]] = None) -> Instance:+        sym = self.lookup_qualified(qualified_name, Context())+        assert sym, "Internal error: attempted to construct unknown type"+        node = sym.node+        assert isinstance(node, TypeInfo)+        if args:+            # TODO: assert len(args) == len(node.defn.type_vars)+            return Instance(node, args)+        return Instance(node, [AnyType(TypeOfAny.special_form)] * len(node.defn.type_vars))++    def named_type_or_none(self, qualified_name: str,+                           args: Optional[List[Type]] = None) -> Optional[Instance]:+        sym = self.lookup_fully_qualified_or_none(qualified_name)+        if not sym:+            return None+        node = sym.node+        if isinstance(node, TypeAlias):+            assert isinstance(node.target, Instance)+            node = node.target.type+        assert isinstance(node, TypeInfo), node+        if args is not None:+            # TODO: assert len(args) == len(node.defn.type_vars)+            return Instance(node, args)+        return Instance(node, [AnyType(TypeOfAny.unannotated)] * len(node.defn.type_vars))++    def visit_import(self, i: Import) -> None:+        for id, as_id in i.ids:+            if as_id is not None:+                self.add_module_symbol(id, as_id, module_public=True, context=i)+            else:+                # Modules imported in a stub file without using 'as x' won't get exported+                module_public = not self.is_stub_file+                base = id.split('.')[0]+                self.add_module_symbol(base, base, module_public=module_public,+                                       context=i, module_hidden=not module_public)+                self.add_submodules_to_parent_modules(id, module_public)++    def add_submodules_to_parent_modules(self, id: str, module_public: bool) -> None:+        """Recursively adds a reference to a newly loaded submodule to its parent.++        When you import a submodule in any way, Python will add a reference to that+        submodule to its parent. So, if you do something like `import A.B` or+        `from A import B` or `from A.B import Foo`, Python will add a reference to+        module A.B to A's namespace.++        Note that this "parent patching" process is completely independent from any+        changes made to the *importer's* namespace. For example, if you have a file+        named `foo.py` where you do `from A.B import Bar`, then foo's namespace will+        be modified to contain a reference to only Bar. Independently, A's namespace+        will be modified to contain a reference to `A.B`.+        """+        while '.' in id:+            parent, child = id.rsplit('.', 1)+            parent_mod = self.modules.get(parent)+            if parent_mod and self.allow_patching(parent_mod, child):+                child_mod = self.modules.get(id)+                if child_mod:+                    sym = SymbolTableNode(MODULE_REF, child_mod,+                                          module_public=module_public,+                                          no_serialize=True)+                else:+                    # Construct a dummy Var with Any type.+                    any_type = AnyType(TypeOfAny.from_unimported_type,+                                       missing_import_name=id)+                    var = Var(child, any_type)+                    var._fullname = child+                    var.is_ready = True+                    var.is_suppressed_import = True+                    sym = SymbolTableNode(GDEF, var,+                                          module_public=module_public,+                                          no_serialize=True)+                parent_mod.names[child] = sym+            id = parent++    def allow_patching(self, parent_mod: MypyFile, child: str) -> bool:+        if child not in parent_mod.names:+            return True+        node = parent_mod.names[child].node+        if isinstance(node, Var) and node.is_suppressed_import:+            return True+        return False++    def add_module_symbol(self, id: str, as_id: str, module_public: bool,+                          context: Context, module_hidden: bool = False) -> None:+        if id in self.modules:+            m = self.modules[id]+            self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m,+                                                   module_public=module_public,+                                                   module_hidden=module_hidden), context)+        else:+            self.add_unknown_symbol(as_id, context, is_import=True, target_name=id)++    def visit_import_from(self, imp: ImportFrom) -> None:+        import_id = self.correct_relative_import(imp)+        self.add_submodules_to_parent_modules(import_id, True)+        module = self.modules.get(import_id)+        for id, as_id in imp.names:+            node = module.names.get(id) if module else None+            node = self.dereference_module_cross_ref(node)++            missing = False+            possible_module_id = import_id + '.' + id++            # If the module does not contain a symbol with the name 'id',+            # try checking if it's a module instead.+            if not node or node.kind == UNBOUND_IMPORTED:+                mod = self.modules.get(possible_module_id)+                if mod is not None:+                    node = SymbolTableNode(MODULE_REF, mod)+                    self.add_submodules_to_parent_modules(possible_module_id, True)+                elif possible_module_id in self.missing_modules:+                    missing = True+            # If it is still not resolved, check for a module level __getattr__+            if (module and not node and (module.is_stub or self.options.python_version >= (3, 7))+                    and '__getattr__' in module.names):+                name = as_id if as_id else id+                if self.type:+                    fullname = self.type.fullname() + "." + name+                else:+                    fullname = self.qualified_name(name)+                gvar = self.create_getattr_var(module.names['__getattr__'], name, fullname)+                if gvar:+                    self.add_symbol(name, gvar, imp)+                    continue+            if node and node.kind != UNBOUND_IMPORTED and not node.module_hidden:+                if not node:+                    # Normalization failed because target is not defined. Avoid duplicate+                    # error messages by marking the imported name as unknown.+                    self.add_unknown_symbol(as_id or id, imp, is_import=True)+                    continue+                imported_id = as_id or id+                existing_symbol = self.globals.get(imported_id)+                if existing_symbol:+                    # Import can redefine a variable. They get special treatment.+                    if self.process_import_over_existing_name(+                            imported_id, existing_symbol, node, imp):+                        continue+                # 'from m import x as x' exports x in a stub file.+                module_public = not self.is_stub_file or as_id is not None+                module_hidden = not module_public and possible_module_id not in self.modules+                symbol = SymbolTableNode(node.kind, node.node,+                                         module_public=module_public,+                                         module_hidden=module_hidden)+                self.add_symbol(imported_id, symbol, imp)+            elif module and not missing:+                # Missing attribute.+                message = "Module '{}' has no attribute '{}'".format(import_id, id)+                extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id))+                if extra:+                    message += " {}".format(extra)+                self.fail(message, imp)+                self.add_unknown_symbol(as_id or id, imp, is_import=True)++                if import_id == 'typing':+                    # The user probably has a missing definition in a test fixture. Let's verify.+                    fullname = 'builtins.{}'.format(id.lower())+                    if (self.lookup_fully_qualified_or_none(fullname) is None and+                            fullname in SUGGESTED_TEST_FIXTURES):+                        # Yes. Generate a helpful note.+                        self.add_fixture_note(fullname, imp)+            else:+                # Missing module.+                missing_name = import_id + '.' + id+                self.add_unknown_symbol(as_id or id, imp, is_import=True, target_name=missing_name)++    def dereference_module_cross_ref(+            self, node: Optional[SymbolTableNode]) -> Optional[SymbolTableNode]:+        """Dereference cross references to other modules (if any).++        If the node is not a cross reference, return it unmodified.+        """+        seen = set()  # type: Set[str]+        # Continue until we reach a node that's nota cross reference (or until we find+        # nothing).+        while node and isinstance(node.node, ImportedName):+            fullname = node.node.fullname()+            if fullname in self.modules:+                # This is a module reference.+                return SymbolTableNode(MODULE_REF, self.modules[fullname])+            if fullname in seen:+                # Looks like a reference cycle. Just break it.+                # TODO: Generate a more specific error message.+                node = None+                break+            node = self.lookup_fully_qualified_or_none(fullname)+            seen.add(fullname)+        return node++    def process_import_over_existing_name(self,+                                          imported_id: str, existing_symbol: SymbolTableNode,+                                          module_symbol: SymbolTableNode,+                                          import_node: ImportBase) -> bool:+        if (existing_symbol.kind in (LDEF, GDEF, MDEF) and+                isinstance(existing_symbol.node, (Var, FuncDef, TypeInfo, Decorator, TypeAlias))):+            # This is a valid import over an existing definition in the file. Construct a dummy+            # assignment that we'll use to type check the import.+            lvalue = NameExpr(imported_id)+            lvalue.kind = existing_symbol.kind+            lvalue.node = existing_symbol.node+            rvalue = NameExpr(imported_id)+            rvalue.kind = module_symbol.kind+            rvalue.node = module_symbol.node+            if isinstance(rvalue.node, TypeAlias):+                # Suppress bogus errors from the dummy assignment if rvalue is an alias.+                # Otherwise mypy may complain that alias is invalid in runtime context.+                rvalue.is_alias_rvalue = True+            assignment = AssignmentStmt([lvalue], rvalue)+            for node in assignment, lvalue, rvalue:+                node.set_line(import_node)+            import_node.assignments.append(assignment)+            return True+        return False++    def add_fixture_note(self, fullname: str, ctx: Context) -> None:+        self.note('Maybe your test fixture does not define "{}"?'.format(fullname), ctx)+        if fullname in SUGGESTED_TEST_FIXTURES:+            self.note(+                'Consider adding [builtins fixtures/{}] to your test description'.format(+                    SUGGESTED_TEST_FIXTURES[fullname]), ctx)++    def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str:+        import_id, ok = correct_relative_import(self.cur_mod_id, node.relative, node.id,+                                                self.cur_mod_node.is_package_init_file())+        if not ok:+            self.fail("Relative import climbs too many namespaces", node)+        return import_id++    def visit_import_all(self, i: ImportAll) -> None:+        i_id = self.correct_relative_import(i)+        if i_id in self.modules:+            m = self.modules[i_id]+            self.add_submodules_to_parent_modules(i_id, True)+            for name, orig_node in m.names.items():+                node = self.dereference_module_cross_ref(orig_node)+                if node is None:+                    continue+                # if '__all__' exists, all nodes not included have had module_public set to+                # False, and we can skip checking '_' because it's been explicitly included.+                if (node.module_public and (not name.startswith('_') or '__all__' in m.names)):+                    existing_symbol = self.lookup_current_scope(name)+                    if existing_symbol:+                        # Import can redefine a variable. They get special treatment.+                        if self.process_import_over_existing_name(+                                name, existing_symbol, node, i):+                            continue+                    symbol = SymbolTableNode(node.kind, node.node)+                    self.add_symbol(name, symbol, i)+                    i.imported_names.append(name)+        else:+            # Don't add any dummy symbols for 'from x import *' if 'x' is unknown.+            pass++    def add_unknown_symbol(self, name: str, context: Context, is_import: bool = False,+                           target_name: Optional[str] = None) -> None:+        var = Var(name)+        if self.options.logical_deps and target_name is not None:+            # This makes it possible to add logical fine-grained dependencies+            # from a missing module. We can't use this by default, since in a+            # few places we assume that the full name points to a real+            # definition, but this name may point to nothing.+            var._fullname = target_name+        elif self.type:+            var._fullname = self.type.fullname() + "." + name+        else:+            var._fullname = self.qualified_name(name)+        var.is_ready = True+        if is_import:+            any_type = AnyType(TypeOfAny.from_unimported_type, missing_import_name=var._fullname)+        else:+            any_type = AnyType(TypeOfAny.from_error)+        var.type = any_type+        var.is_suppressed_import = is_import+        self.add_symbol(name, SymbolTableNode(GDEF, var), context)++    #+    # Statements+    #++    def visit_block(self, b: Block) -> None:+        if b.is_unreachable:+            return+        self.block_depth[-1] += 1+        for s in b.body:+            self.accept(s)+        self.block_depth[-1] -= 1++    def visit_block_maybe(self, b: Optional[Block]) -> None:+        if b:+            self.visit_block(b)++    def type_analyzer(self, *,+                      tvar_scope: Optional[TypeVarScope] = None,+                      allow_tuple_literal: bool = False,+                      allow_unbound_tvars: bool = False,+                      third_pass: bool = False) -> TypeAnalyser:+        if tvar_scope is None:+            tvar_scope = self.tvar_scope+        tpan = TypeAnalyser(self,+                            tvar_scope,+                            self.plugin,+                            self.options,+                            self.is_typeshed_stub_file,+                            allow_unbound_tvars=allow_unbound_tvars,+                            allow_tuple_literal=allow_tuple_literal,+                            allow_unnormalized=self.is_stub_file,+                            third_pass=third_pass)+        tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic())+        tpan.global_scope = not self.type and not self.function_stack+        return tpan++    def anal_type(self, t: Type, *,+                  tvar_scope: Optional[TypeVarScope] = None,+                  allow_tuple_literal: bool = False,+                  allow_unbound_tvars: bool = False,+                  third_pass: bool = False) -> Type:+        a = self.type_analyzer(tvar_scope=tvar_scope,+                               allow_unbound_tvars=allow_unbound_tvars,+                               allow_tuple_literal=allow_tuple_literal,+                               third_pass=third_pass)+        typ = t.accept(a)+        self.add_type_alias_deps(a.aliases_used)+        return typ++    def add_type_alias_deps(self, aliases_used: Iterable[str],+                            target: Optional[str] = None) -> None:+        """Add full names of type aliases on which the current node depends.++        This is used by fine-grained incremental mode to re-check the corresponding nodes.+        If `target` is None, then the target node used will be the current scope.+        """+        if not aliases_used:+            # A basic optimization to avoid adding targets with no dependencies to+            # the `alias_deps` dict.+            return+        if target is None:+            target = self.scope.current_target()+        self.cur_mod_node.alias_deps[target].update(aliases_used)++    def visit_assignment_stmt(self, s: AssignmentStmt) -> None:+        for lval in s.lvalues:+            self.analyze_lvalue(lval, explicit_type=s.type is not None)+        self.check_classvar(s)+        s.rvalue.accept(self)+        if s.type:+            allow_tuple_literal = isinstance(s.lvalues[-1], TupleExpr)+            s.type = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal)+            if (self.type and self.type.is_protocol and isinstance(lval, NameExpr) and+                    isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs):+                        if isinstance(lval.node, Var):+                            lval.node.is_abstract_var = True+        else:+            if (any(isinstance(lv, NameExpr) and lv.is_inferred_def for lv in s.lvalues) and+                    self.type and self.type.is_protocol and not self.is_func_scope()):+                self.fail('All protocol members must have explicitly declared types', s)+            # Set the type if the rvalue is a simple literal (even if the above error occurred).+            if len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr):+                if s.lvalues[0].is_inferred_def:+                    s.type = self.analyze_simple_literal_type(s.rvalue)+        if s.type:+            # Store type into nodes.+            for lvalue in s.lvalues:+                self.store_declared_types(lvalue, s.type)+        self.check_and_set_up_type_alias(s)+        self.newtype_analyzer.process_newtype_declaration(s)+        self.process_typevar_declaration(s)+        self.named_tuple_analyzer.process_namedtuple_definition(s, self.is_func_scope())+        self.typed_dict_analyzer.process_typeddict_definition(s, self.is_func_scope())+        self.enum_call_analyzer.process_enum_call(s, self.is_func_scope())+        if not s.type:+            self.process_module_assignment(s.lvalues, s.rvalue, s)++        if (len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr) and+                s.lvalues[0].name == '__all__' and s.lvalues[0].kind == GDEF and+                isinstance(s.rvalue, (ListExpr, TupleExpr))):+            self.add_exports(s.rvalue.items)++    def analyze_simple_literal_type(self, rvalue: Expression) -> Optional[Type]:+        """Return builtins.int if rvalue is an int literal, etc."""+        if self.options.semantic_analysis_only or self.function_stack:+            # Skip this if we're only doing the semantic analysis pass.+            # This is mostly to avoid breaking unit tests.+            # Also skip inside a function; this is to avoid confusing+            # the code that handles dead code due to isinstance()+            # inside type variables with value restrictions (like+            # AnyStr).+            return None+        if isinstance(rvalue, IntExpr):+            return self.named_type_or_none('builtins.int')+        if isinstance(rvalue, FloatExpr):+            return self.named_type_or_none('builtins.float')+        if isinstance(rvalue, StrExpr):+            return self.named_type_or_none('builtins.str')+        if isinstance(rvalue, BytesExpr):+            return self.named_type_or_none('builtins.bytes')+        if isinstance(rvalue, UnicodeExpr):+            return self.named_type_or_none('builtins.unicode')+        return None++    def analyze_alias(self, rvalue: Expression) -> Tuple[Optional[Type], List[str],+                                                         Set[str], List[str]]:+        """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable).++        If yes, return the corresponding type, a list of+        qualified type variable names for generic aliases, a set of names the alias depends on,+        and a list of type variables if the alias is generic.+        An schematic example for the dependencies:+            A = int+            B = str+            analyze_alias(Dict[A, B])[2] == {'__main__.A', '__main__.B'}+        """+        dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic())+        global_scope = not self.type and not self.function_stack+        res = analyze_type_alias(rvalue,+                                 self,+                                 self.tvar_scope,+                                 self.plugin,+                                 self.options,+                                 self.is_typeshed_stub_file,+                                 allow_unnormalized=self.is_stub_file,+                                 in_dynamic_func=dynamic,+                                 global_scope=global_scope)+        typ = None  # type: Optional[Type]+        if res:+            typ, depends_on = res+            found_type_vars = typ.accept(TypeVariableQuery(self.lookup_qualified, self.tvar_scope))+            alias_tvars = [name for (name, node) in found_type_vars]+            qualified_tvars = [node.fullname() for (name, node) in found_type_vars]+        else:+            alias_tvars = []+            depends_on = set()+            qualified_tvars = []+        return typ, alias_tvars, depends_on, qualified_tvars++    def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None:+        """Check if assignment creates a type alias and set it up as needed.++        For simple aliases like L = List we use a simpler mechanism, just copying TypeInfo.+        For subscripted (including generic) aliases the resulting types are stored+        in rvalue.analyzed.+        """+        lvalue = s.lvalues[0]+        if len(s.lvalues) > 1 or not isinstance(lvalue, NameExpr):+            # First rule: Only simple assignments like Alias = ... create aliases.+            return+        if s.type:+            # Second rule: Explicit type (cls: Type[A] = A) always creates variable, not alias.+            return+        non_global_scope = self.type or self.is_func_scope()+        if isinstance(s.rvalue, RefExpr) and non_global_scope and lvalue.is_inferred_def:+            # Third rule: Non-subscripted right hand side creates a variable+            # at class and function scopes. For example:+            #+            #   class Model:+            #       ...+            #   class C:+            #       model = Model # this is automatically a variable with type 'Type[Model]'+            #+            # without this rule, this typical use case will require a lot of explicit+            # annotations (see the second rule).+            return+        rvalue = s.rvalue+        res, alias_tvars, depends_on, qualified_tvars = self.analyze_alias(rvalue)+        if not res:+            return+        s.is_alias_def = True+        node = self.lookup(lvalue.name, lvalue)+        assert node is not None+        assert node.node is not None+        self.add_type_alias_deps(depends_on)+        # In addition to the aliases used, we add deps on unbound+        # type variables, since they are erased from target type.+        self.add_type_alias_deps(qualified_tvars)+        # The above are only direct deps on other aliases.+        # For subscripted aliases, type deps from expansion are added in deps.py+        # (because the type is stored)+        if not lvalue.is_inferred_def:+            # Type aliases can't be re-defined.+            if isinstance(node.node, (TypeAlias, TypeInfo)):+                self.fail('Cannot assign multiple types to name "{}"'+                          ' without an explicit "Type[...]" annotation'+                          .format(lvalue.name), lvalue)+            return+        check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg,+                               context=s)+        # when this type alias gets "inlined", the Any is not explicit anymore,+        # so we need to replace it with non-explicit Anys+        res = make_any_non_explicit(res)+        no_args = isinstance(res, Instance) and not res.args+        if isinstance(s.rvalue, (IndexExpr, CallExpr)):  # CallExpr is for `void = type(None)`+            s.rvalue.analyzed = TypeAliasExpr(res, alias_tvars, no_args)+            s.rvalue.analyzed.line = s.line+            # we use the column from resulting target, to get better location for errors+            s.rvalue.analyzed.column = res.column+        elif isinstance(s.rvalue, RefExpr):+            s.rvalue.is_alias_rvalue = True+        node.node = TypeAlias(res, node.node.fullname(), s.line, s.column,+                              alias_tvars=alias_tvars, no_args=no_args)+        if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias):+            node.node.normalized = rvalue.node.normalized++    def analyze_lvalue(self, lval: Lvalue, nested: bool = False,+                       add_global: bool = False,+                       explicit_type: bool = False) -> None:+        """Analyze an lvalue or assignment target.++        Args:+            lval: The target lvalue+            nested: If true, the lvalue is within a tuple or list lvalue expression+            add_global: Add name to globals table only if this is true (used in first pass)+            explicit_type: Assignment has type annotation+        """+        if isinstance(lval, NameExpr):+            # Top-level definitions within some statements (at least while) are+            # not handled in the first pass, so they have to be added now.+            nested_global = (not self.is_func_scope() and+                             self.block_depth[-1] > 0 and+                             not self.type)+            if (add_global or nested_global) and lval.name not in self.globals:+                # Define new global name.+                v = Var(lval.name)+                v.set_line(lval)+                v._fullname = self.qualified_name(lval.name)+                v.is_ready = False  # Type not inferred yet+                lval.node = v+                lval.is_new_def = True+                lval.is_inferred_def = True+                lval.kind = GDEF+                lval.fullname = v._fullname+                self.globals[lval.name] = SymbolTableNode(GDEF, v)+            elif isinstance(lval.node, Var) and lval.is_new_def:+                if lval.kind == GDEF:+                    # Since the is_new_def flag is set, this must have been analyzed+                    # already in the first pass and added to the symbol table.+                    # An exception is typing module with incomplete test fixtures.+                    assert lval.node.name() in self.globals or self.cur_mod_id == 'typing'+            elif (self.locals[-1] is not None and lval.name not in self.locals[-1] and+                  lval.name not in self.global_decls[-1] and+                  lval.name not in self.nonlocal_decls[-1]):+                # Define new local name.+                v = Var(lval.name)+                v.set_line(lval)+                lval.node = v+                lval.is_new_def = True+                lval.is_inferred_def = True+                lval.kind = LDEF+                lval.fullname = lval.name+                self.add_local(v, lval)+                if lval.name == '_':+                    # Special case for assignment to local named '_': always infer 'Any'.+                    typ = AnyType(TypeOfAny.special_form)+                    self.store_declared_types(lval, typ)+            elif not self.is_func_scope() and (self.type and+                                               lval.name not in self.type.names):+                # Define a new attribute within class body.+                v = Var(lval.name)+                v.info = self.type+                v.is_initialized_in_class = True+                v.is_inferred = not explicit_type+                v.set_line(lval)+                v._fullname = self.qualified_name(lval.name)+                lval.node = v+                lval.is_new_def = True+                lval.is_inferred_def = True+                lval.kind = MDEF+                lval.fullname = lval.name+                self.type.names[lval.name] = SymbolTableNode(MDEF, v)+            elif explicit_type:+                # Don't re-bind types+                global_def = self.globals.get(lval.name)+                if self.locals:+                    locals_last = self.locals[-1]+                    if locals_last:+                        local_def = locals_last.get(lval.name)+                    else:+                        local_def = None+                else:+                    local_def = None+                type_def = self.type.names.get(lval.name) if self.type else None++                original_def = global_def or local_def or type_def+                self.name_already_defined(lval.name, lval, original_def)+            else:+                # Bind to an existing name.+                lval.accept(self)+                self.check_lvalue_validity(lval.node, lval)+        elif isinstance(lval, MemberExpr):+            if not add_global:+                self.analyze_member_lvalue(lval, explicit_type)+            if explicit_type and not self.is_self_member_ref(lval):+                self.fail('Type cannot be declared in assignment to non-self '+                          'attribute', lval)+        elif isinstance(lval, IndexExpr):+            if explicit_type:+                self.fail('Unexpected type declaration', lval)+            if not add_global:+                lval.accept(self)+        elif isinstance(lval, TupleExpr):+            items = lval.items+            if len(items) == 0 and isinstance(lval, TupleExpr):+                self.fail("can't assign to ()", lval)+            self.analyze_tuple_or_list_lvalue(lval, add_global, explicit_type)+        elif isinstance(lval, StarExpr):+            if nested:+                self.analyze_lvalue(lval.expr, nested, add_global, explicit_type)+            else:+                self.fail('Starred assignment target must be in a list or tuple', lval)+        else:+            self.fail('Invalid assignment target', lval)++    def analyze_tuple_or_list_lvalue(self, lval: TupleExpr,+                                     add_global: bool = False,+                                     explicit_type: bool = False) -> None:+        """Analyze an lvalue or assignment target that is a list or tuple."""+        items = lval.items+        star_exprs = [item for item in items if isinstance(item, StarExpr)]++        if len(star_exprs) > 1:+            self.fail('Two starred expressions in assignment', lval)+        else:+            if len(star_exprs) == 1:+                star_exprs[0].valid = True+            for i in items:+                self.analyze_lvalue(i, nested=True, add_global=add_global,+                                    explicit_type = explicit_type)++    def analyze_member_lvalue(self, lval: MemberExpr, explicit_type: bool = False) -> None:+        lval.accept(self)+        if self.is_self_member_ref(lval):+            assert self.type, "Self member outside a class"+            cur_node = self.type.names.get(lval.name, None)+            node = self.type.get(lval.name)+            # If the attribute of self is not defined in superclasses, create a new Var, ...+            if ((node is None or isinstance(node.node, Var) and node.node.is_abstract_var) or+                    # ... also an explicit declaration on self also creates a new Var.+                    (cur_node is None and explicit_type)):+                if self.type.is_protocol and node is None:+                    self.fail("Protocol members cannot be defined via assignment to self", lval)+                else:+                    # Implicit attribute definition in __init__.+                    lval.is_new_def = True+                    lval.is_inferred_def = True+                    v = Var(lval.name)+                    v.set_line(lval)+                    v._fullname = self.qualified_name(lval.name)+                    v.info = self.type+                    v.is_ready = False+                    lval.def_var = v+                    lval.node = v+                    # TODO: should we also set lval.kind = MDEF?+                    self.type.names[lval.name] = SymbolTableNode(MDEF, v, implicit=True)+        self.check_lvalue_validity(lval.node, lval)++    def is_self_member_ref(self, memberexpr: MemberExpr) -> bool:+        """Does memberexpr to refer to an attribute of self?"""+        if not isinstance(memberexpr.expr, NameExpr):+            return False+        node = memberexpr.expr.node+        return isinstance(node, Var) and node.is_self++    def check_lvalue_validity(self, node: Union[Expression, SymbolNode, None],+                              ctx: Context) -> None:+        if isinstance(node, TypeVarExpr):+            self.fail('Invalid assignment target', ctx)+        elif isinstance(node, TypeInfo):+            self.fail(CANNOT_ASSIGN_TO_TYPE, ctx)++    def store_declared_types(self, lvalue: Lvalue, typ: Type) -> None:+        if isinstance(typ, StarType) and not isinstance(lvalue, StarExpr):+            self.fail('Star type only allowed for starred expressions', lvalue)+        if isinstance(lvalue, RefExpr):+            lvalue.is_inferred_def = False+            if isinstance(lvalue.node, Var):+                var = lvalue.node+                var.type = typ+                var.is_ready = True+            # If node is not a variable, we'll catch it elsewhere.+        elif isinstance(lvalue, TupleExpr):+            if isinstance(typ, TupleType):+                if len(lvalue.items) != len(typ.items):+                    self.fail('Incompatible number of tuple items', lvalue)+                    return+                for item, itemtype in zip(lvalue.items, typ.items):+                    self.store_declared_types(item, itemtype)+            else:+                self.fail('Tuple type expected for multiple variables',+                          lvalue)+        elif isinstance(lvalue, StarExpr):+            # Historical behavior for the old parser+            if isinstance(typ, StarType):+                self.store_declared_types(lvalue.expr, typ.type)+            else:+                self.store_declared_types(lvalue.expr, typ)+        else:+            # This has been flagged elsewhere as an error, so just ignore here.+            pass++    def process_typevar_declaration(self, s: AssignmentStmt) -> None:+        """Check if s declares a TypeVar; it yes, store it in symbol table."""+        call = self.get_typevar_declaration(s)+        if not call:+            return++        lvalue = s.lvalues[0]+        assert isinstance(lvalue, NameExpr)+        name = lvalue.name+        if not lvalue.is_inferred_def:+            if s.type:+                self.fail("Cannot declare the type of a type variable", s)+            else:+                self.fail("Cannot redefine '%s' as a type variable" % name, s)+            return++        if not self.check_typevar_name(call, name, s):+            return++        # Constraining types+        n_values = call.arg_kinds[1:].count(ARG_POS)+        values = self.analyze_types(call.args[1:1 + n_values])++        res = self.process_typevar_parameters(call.args[1 + n_values:],+                                              call.arg_names[1 + n_values:],+                                              call.arg_kinds[1 + n_values:],+                                              n_values,+                                              s)+        if res is None:+            return+        variance, upper_bound = res++        if self.options.disallow_any_unimported:+            for idx, constraint in enumerate(values, start=1):+                if has_any_from_unimported_type(constraint):+                    prefix = "Constraint {}".format(idx)+                    self.msg.unimported_type_becomes_any(prefix, constraint, s)++            if has_any_from_unimported_type(upper_bound):+                prefix = "Upper bound of type variable"+                self.msg.unimported_type_becomes_any(prefix, upper_bound, s)++        for t in values + [upper_bound]:+            check_for_explicit_any(t, self.options, self.is_typeshed_stub_file, self.msg,+                                   context=s)+        # Yes, it's a valid type variable definition! Add it to the symbol table.+        node = self.lookup(name, s)+        assert node is not None+        assert node.fullname is not None+        node.kind = TVAR+        TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance)+        TypeVar.line = call.line+        call.analyzed = TypeVar+        node.node = TypeVar++    def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool:+        if len(call.args) < 1:+            self.fail("Too few arguments for TypeVar()", context)+            return False+        if (not isinstance(call.args[0], (StrExpr, BytesExpr, UnicodeExpr))+                or not call.arg_kinds[0] == ARG_POS):+            self.fail("TypeVar() expects a string literal as first argument", context)+            return False+        elif call.args[0].value != name:+            msg = "String argument 1 '{}' to TypeVar(...) does not match variable name '{}'"+            self.fail(msg.format(call.args[0].value, name), context)+            return False+        return True++    def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]:+        """Returns the TypeVar() call expression if `s` is a type var declaration+        or None otherwise.+        """+        if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):+            return None+        if not isinstance(s.rvalue, CallExpr):+            return None+        call = s.rvalue+        callee = call.callee+        if not isinstance(callee, RefExpr):+            return None+        if callee.fullname != 'typing.TypeVar':+            return None+        return call++    def process_typevar_parameters(self, args: List[Expression],+                                   names: List[Optional[str]],+                                   kinds: List[int],+                                   num_values: int,+                                   context: Context) -> Optional[Tuple[int, Type]]:+        has_values = (num_values > 0)+        covariant = False+        contravariant = False+        upper_bound = self.object_type()   # type: Type+        for param_value, param_name, param_kind in zip(args, names, kinds):+            if not param_kind == ARG_NAMED:+                self.fail("Unexpected argument to TypeVar()", context)+                return None+            if param_name == 'covariant':+                if isinstance(param_value, NameExpr):+                    if param_value.name == 'True':+                        covariant = True+                    else:+                        self.fail("TypeVar 'covariant' may only be 'True'", context)+                        return None+                else:+                    self.fail("TypeVar 'covariant' may only be 'True'", context)+                    return None+            elif param_name == 'contravariant':+                if isinstance(param_value, NameExpr):+                    if param_value.name == 'True':+                        contravariant = True+                    else:+                        self.fail("TypeVar 'contravariant' may only be 'True'", context)+                        return None+                else:+                    self.fail("TypeVar 'contravariant' may only be 'True'", context)+                    return None+            elif param_name == 'bound':+                if has_values:+                    self.fail("TypeVar cannot have both values and an upper bound", context)+                    return None+                try:+                    upper_bound = self.expr_to_analyzed_type(param_value)+                except TypeTranslationError:+                    self.fail("TypeVar 'bound' must be a type", param_value)+                    return None+            elif param_name == 'values':+                # Probably using obsolete syntax with values=(...). Explain the current syntax.+                self.fail("TypeVar 'values' argument not supported", context)+                self.fail("Use TypeVar('T', t, ...) instead of TypeVar('T', values=(t, ...))",+                          context)+                return None+            else:+                self.fail("Unexpected argument to TypeVar(): {}".format(param_name), context)+                return None++        if covariant and contravariant:+            self.fail("TypeVar cannot be both covariant and contravariant", context)+            return None+        elif num_values == 1:+            self.fail("TypeVar cannot have only a single constraint", context)+            return None+        elif covariant:+            variance = COVARIANT+        elif contravariant:+            variance = CONTRAVARIANT+        else:+            variance = INVARIANT+        return (variance, upper_bound)++    def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance) -> TypeInfo:+        class_def = ClassDef(name, Block([]))+        class_def.fullname = self.qualified_name(name)++        info = TypeInfo(SymbolTable(), class_def, self.cur_mod_id)+        class_def.info = info+        mro = basetype_or_fallback.type.mro+        if not mro:+            # Forward reference, MRO should be recalculated in third pass.+            mro = [basetype_or_fallback.type, self.object_type().type]+        info.mro = [info] + mro+        info.bases = [basetype_or_fallback]+        return info++    def analyze_types(self, items: List[Expression]) -> List[Type]:+        result = []  # type: List[Type]+        for node in items:+            try:+                result.append(self.anal_type(expr_to_unanalyzed_type(node)))+            except TypeTranslationError:+                self.fail('Type expected', node)+                result.append(AnyType(TypeOfAny.from_error))+        return result++    def parse_bool(self, expr: Expression) -> Optional[bool]:+        if isinstance(expr, NameExpr):+            if expr.fullname == 'builtins.True':+                return True+            if expr.fullname == 'builtins.False':+                return False+        return None++    def check_classvar(self, s: AssignmentStmt) -> None:+        lvalue = s.lvalues[0]+        if len(s.lvalues) != 1 or not isinstance(lvalue, RefExpr):+            return+        if not s.type or not self.is_classvar(s.type):+            return+        if self.is_class_scope() and isinstance(lvalue, NameExpr):+            node = lvalue.node+            if isinstance(node, Var):+                node.is_classvar = True+        elif not isinstance(lvalue, MemberExpr) or self.is_self_member_ref(lvalue):+            # In case of member access, report error only when assigning to self+            # Other kinds of member assignments should be already reported+            self.fail_invalid_classvar(lvalue)++    def is_classvar(self, typ: Type) -> bool:+        if not isinstance(typ, UnboundType):+            return False+        sym = self.lookup_qualified(typ.name, typ)+        if not sym or not sym.node:+            return False+        return sym.node.fullname() == 'typing.ClassVar'++    def fail_invalid_classvar(self, context: Context) -> None:+        self.fail('ClassVar can only be used for assignments in class body', context)++    def process_module_assignment(self, lvals: List[Lvalue], rval: Expression,+                                  ctx: AssignmentStmt) -> None:+        """Propagate module references across assignments.++        Recursively handles the simple form of iterable unpacking; doesn't+        handle advanced unpacking with *rest, dictionary unpacking, etc.++        In an expression like x = y = z, z is the rval and lvals will be [x,+        y].++        """+        if (isinstance(rval, (TupleExpr, ListExpr))+                and all(isinstance(v, TupleExpr) for v in lvals)):+            # rval and all lvals are either list or tuple, so we are dealing+            # with unpacking assignment like `x, y = a, b`. Mypy didn't+            # understand our all(isinstance(...)), so cast them as TupleExpr+            # so mypy knows it is safe to access their .items attribute.+            seq_lvals = cast(List[TupleExpr], lvals)+            # given an assignment like:+            #     (x, y) = (m, n) = (a, b)+            # we now have:+            #     seq_lvals = [(x, y), (m, n)]+            #     seq_rval = (a, b)+            # We now zip this into:+            #     elementwise_assignments = [(a, x, m), (b, y, n)]+            # where each elementwise assignment includes one element of rval and the+            # corresponding element of each lval. Basically we unpack+            #     (x, y) = (m, n) = (a, b)+            # into elementwise assignments+            #     x = m = a+            #     y = n = b+            # and then we recursively call this method for each of those assignments.+            # If the rval and all lvals are not all of the same length, zip will just ignore+            # extra elements, so no error will be raised here; mypy will later complain+            # about the length mismatch in type-checking.+            elementwise_assignments = zip(rval.items, *[v.items for v in seq_lvals])+            # TODO: use 'for rv, *lvs in' once mypyc supports it+            for part in elementwise_assignments:+                rv, lvs = part[0], list(part[1:])+                self.process_module_assignment(lvs, rv, ctx)+        elif isinstance(rval, RefExpr):+            rnode = self.lookup_type_node(rval)+            if rnode and rnode.kind == MODULE_REF:+                for lval in lvals:+                    if not isinstance(lval, NameExpr):+                        continue+                    # respect explicitly annotated type+                    if (isinstance(lval.node, Var) and lval.node.type is not None):+                        continue+                    lnode = self.lookup(lval.name, ctx)+                    if lnode:+                        if lnode.kind == MODULE_REF and lnode.node is not rnode.node:+                            self.fail(+                                "Cannot assign multiple modules to name '{}' "+                                "without explicit 'types.ModuleType' annotation".format(lval.name),+                                ctx)+                        # never create module alias except on initial var definition+                        elif lval.is_inferred_def:+                            lnode.kind = MODULE_REF+                            lnode.node = rnode.node++    def visit_decorator(self, dec: Decorator) -> None:+        for d in dec.decorators:+            d.accept(self)+        removed = []  # type: List[int]+        no_type_check = False+        for i, d in enumerate(dec.decorators):+            # A bunch of decorators are special cased here.+            if refers_to_fullname(d, 'abc.abstractmethod'):+                removed.append(i)+                dec.func.is_abstract = True+                self.check_decorated_function_is_method('abstractmethod', dec)+            elif (refers_to_fullname(d, 'asyncio.coroutines.coroutine') or+                  refers_to_fullname(d, 'types.coroutine')):+                removed.append(i)+                dec.func.is_awaitable_coroutine = True+            elif refers_to_fullname(d, 'builtins.staticmethod'):+                removed.append(i)+                dec.func.is_static = True+                dec.var.is_staticmethod = True+                self.check_decorated_function_is_method('staticmethod', dec)+            elif refers_to_fullname(d, 'builtins.classmethod'):+                removed.append(i)+                dec.func.is_class = True+                dec.var.is_classmethod = True+                self.check_decorated_function_is_method('classmethod', dec)+            elif (refers_to_fullname(d, 'builtins.property') or+                  refers_to_fullname(d, 'abc.abstractproperty')):+                removed.append(i)+                dec.func.is_property = True+                dec.var.is_property = True+                if refers_to_fullname(d, 'abc.abstractproperty'):+                    dec.func.is_abstract = True+                self.check_decorated_function_is_method('property', dec)+                if len(dec.func.arguments) > 1:+                    self.fail('Too many arguments', dec.func)+            elif refers_to_fullname(d, 'typing.no_type_check'):+                dec.var.type = AnyType(TypeOfAny.special_form)+                no_type_check = True+        for i in reversed(removed):+            del dec.decorators[i]+        if not dec.is_overload or dec.var.is_property:+            if self.is_func_scope():+                self.add_symbol(dec.var.name(), SymbolTableNode(LDEF, dec),+                                dec)+            elif self.type:+                dec.var.info = self.type+                dec.var.is_initialized_in_class = True+                self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec),+                                dec)+        if not no_type_check and self.recurse_into_functions:+            dec.func.accept(self)+        if dec.decorators and dec.var.is_property:+            self.fail('Decorated property not supported', dec)++    def check_decorated_function_is_method(self, decorator: str,+                                           context: Context) -> None:+        if not self.type or self.is_func_scope():+            self.fail("'%s' used with a non-method" % decorator, context)++    def visit_expression_stmt(self, s: ExpressionStmt) -> None:+        s.expr.accept(self)++    def visit_return_stmt(self, s: ReturnStmt) -> None:+        if not self.is_func_scope():+            self.fail("'return' outside function", s)+        if s.expr:+            s.expr.accept(self)++    def visit_raise_stmt(self, s: RaiseStmt) -> None:+        if s.expr:+            s.expr.accept(self)+        if s.from_expr:+            s.from_expr.accept(self)++    def visit_assert_stmt(self, s: AssertStmt) -> None:+        if s.expr:+            s.expr.accept(self)+        if s.msg:+            s.msg.accept(self)++    def visit_operator_assignment_stmt(self,+                                       s: OperatorAssignmentStmt) -> None:+        s.lvalue.accept(self)+        s.rvalue.accept(self)+        if (isinstance(s.lvalue, NameExpr) and s.lvalue.name == '__all__' and+                s.lvalue.kind == GDEF and isinstance(s.rvalue, (ListExpr, TupleExpr))):+            self.add_exports(s.rvalue.items)++    def visit_while_stmt(self, s: WhileStmt) -> None:+        s.expr.accept(self)+        self.loop_depth += 1+        s.body.accept(self)+        self.loop_depth -= 1+        self.visit_block_maybe(s.else_body)++    def visit_for_stmt(self, s: ForStmt) -> None:+        s.expr.accept(self)++        # Bind index variables and check if they define new names.+        self.analyze_lvalue(s.index, explicit_type=s.index_type is not None)+        if s.index_type:+            if self.is_classvar(s.index_type):+                self.fail_invalid_classvar(s.index)+            allow_tuple_literal = isinstance(s.index, TupleExpr)+            s.index_type = self.anal_type(s.index_type, allow_tuple_literal=allow_tuple_literal)+            self.store_declared_types(s.index, s.index_type)++        self.loop_depth += 1+        self.visit_block(s.body)+        self.loop_depth -= 1++        self.visit_block_maybe(s.else_body)++    def visit_break_stmt(self, s: BreakStmt) -> None:+        if self.loop_depth == 0:+            self.fail("'break' outside loop", s, True, blocker=True)++    def visit_continue_stmt(self, s: ContinueStmt) -> None:+        if self.loop_depth == 0:+            self.fail("'continue' outside loop", s, True, blocker=True)++    def visit_if_stmt(self, s: IfStmt) -> None:+        infer_reachability_of_if_statement(s, self.options)+        for i in range(len(s.expr)):+            s.expr[i].accept(self)+            self.visit_block(s.body[i])+        self.visit_block_maybe(s.else_body)++    def visit_try_stmt(self, s: TryStmt) -> None:+        self.analyze_try_stmt(s, self)++    def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor[None],+                         add_global: bool = False) -> None:+        s.body.accept(visitor)+        for type, var, handler in zip(s.types, s.vars, s.handlers):+            if type:+                type.accept(visitor)+            if var:+                self.analyze_lvalue(var, add_global=add_global)+            handler.accept(visitor)+        if s.else_body:+            s.else_body.accept(visitor)+        if s.finally_body:+            s.finally_body.accept(visitor)++    def visit_with_stmt(self, s: WithStmt) -> None:+        types = []  # type: List[Type]++        if s.target_type:+            actual_targets = [t for t in s.target if t is not None]+            if len(actual_targets) == 0:+                # We have a type for no targets+                self.fail('Invalid type comment', s)+            elif len(actual_targets) == 1:+                # We have one target and one type+                types = [s.target_type]+            elif isinstance(s.target_type, TupleType):+                # We have multiple targets and multiple types+                if len(actual_targets) == len(s.target_type.items):+                    types = s.target_type.items+                else:+                    # But it's the wrong number of items+                    self.fail('Incompatible number of types for `with` targets', s)+            else:+                # We have multiple targets and one type+                self.fail('Multiple types expected for multiple `with` targets', s)++        new_types = []  # type: List[Type]+        for e, n in zip(s.expr, s.target):+            e.accept(self)+            if n:+                self.analyze_lvalue(n, explicit_type=s.target_type is not None)++                # Since we have a target, pop the next type from types+                if types:+                    t = types.pop(0)+                    if self.is_classvar(t):+                        self.fail_invalid_classvar(n)+                    allow_tuple_literal = isinstance(n, TupleExpr)+                    t = self.anal_type(t, allow_tuple_literal=allow_tuple_literal)+                    new_types.append(t)+                    self.store_declared_types(n, t)++        # Reverse the logic above to correctly reassign target_type+        if new_types:+            if len(s.target) == 1:+                s.target_type = new_types[0]+            elif isinstance(s.target_type, TupleType):+                s.target_type = s.target_type.copy_modified(items=new_types)++        self.visit_block(s.body)++    def visit_del_stmt(self, s: DelStmt) -> None:+        s.expr.accept(self)+        if not self.is_valid_del_target(s.expr):+            self.fail('Invalid delete target', s)++    def is_valid_del_target(self, s: Expression) -> bool:+        if isinstance(s, (IndexExpr, NameExpr, MemberExpr)):+            return True+        elif isinstance(s, TupleExpr):+            return all(self.is_valid_del_target(item) for item in s.items)+        else:+            return False++    def visit_global_decl(self, g: GlobalDecl) -> None:+        for name in g.names:+            if name in self.nonlocal_decls[-1]:+                self.fail("Name '{}' is nonlocal and global".format(name), g)+            self.global_decls[-1].add(name)++    def visit_nonlocal_decl(self, d: NonlocalDecl) -> None:+        if not self.is_func_scope():+            self.fail("nonlocal declaration not allowed at module level", d)+        else:+            for name in d.names:+                for table in reversed(self.locals[:-1]):+                    if table is not None and name in table:+                        break+                else:+                    self.fail("No binding for nonlocal '{}' found".format(name), d)++                if self.locals[-1] is not None and name in self.locals[-1]:+                    self.fail("Name '{}' is already defined in local "+                              "scope before nonlocal declaration".format(name), d)++                if name in self.global_decls[-1]:+                    self.fail("Name '{}' is nonlocal and global".format(name), d)+                self.nonlocal_decls[-1].add(name)++    def visit_print_stmt(self, s: PrintStmt) -> None:+        for arg in s.args:+            arg.accept(self)+        if s.target:+            s.target.accept(self)++    def visit_exec_stmt(self, s: ExecStmt) -> None:+        s.expr.accept(self)+        if s.globals:+            s.globals.accept(self)+        if s.locals:+            s.locals.accept(self)++    #+    # Expressions+    #++    def visit_name_expr(self, expr: NameExpr) -> None:+        n = self.lookup(expr.name, expr)+        if n:+            if n.kind == TVAR and self.tvar_scope.get_binding(n):+                self.fail("'{}' is a type variable and only valid in type "+                          "context".format(expr.name), expr)+            else:+                expr.kind = n.kind+                expr.node = n.node+                expr.fullname = n.fullname++    def visit_super_expr(self, expr: SuperExpr) -> None:+        if not self.type:+            self.fail('"super" used outside class', expr)+            return+        expr.info = self.type+        for arg in expr.call.args:+            arg.accept(self)++    def visit_tuple_expr(self, expr: TupleExpr) -> None:+        for item in expr.items:+            if isinstance(item, StarExpr):+                item.valid = True+            item.accept(self)++    def visit_list_expr(self, expr: ListExpr) -> None:+        for item in expr.items:+            if isinstance(item, StarExpr):+                item.valid = True+            item.accept(self)++    def visit_set_expr(self, expr: SetExpr) -> None:+        for item in expr.items:+            if isinstance(item, StarExpr):+                item.valid = True+            item.accept(self)++    def visit_dict_expr(self, expr: DictExpr) -> None:+        for key, value in expr.items:+            if key is not None:+                key.accept(self)+            value.accept(self)++    def visit_star_expr(self, expr: StarExpr) -> None:+        if not expr.valid:+            # XXX TODO Change this error message+            self.fail('Can use starred expression only as assignment target', expr)+        else:+            expr.expr.accept(self)++    def visit_yield_from_expr(self, e: YieldFromExpr) -> None:+        if not self.is_func_scope():  # not sure+            self.fail("'yield from' outside function", e, True, blocker=True)+        else:+            if self.function_stack[-1].is_coroutine:+                self.fail("'yield from' in async function", e, True, blocker=True)+            else:+                self.function_stack[-1].is_generator = True+        if e.expr:+            e.expr.accept(self)++    def visit_call_expr(self, expr: CallExpr) -> None:+        """Analyze a call expression.++        Some call expressions are recognized as special forms, including+        cast(...).+        """+        if expr.analyzed:+            return+        expr.callee.accept(self)+        if refers_to_fullname(expr.callee, 'typing.cast'):+            # Special form cast(...).+            if not self.check_fixed_args(expr, 2, 'cast'):+                return+            # Translate first argument to an unanalyzed type.+            try:+                target = expr_to_unanalyzed_type(expr.args[0])+            except TypeTranslationError:+                self.fail('Cast target is not a type', expr)+                return+            # Piggyback CastExpr object to the CallExpr object; it takes+            # precedence over the CallExpr semantics.+            expr.analyzed = CastExpr(expr.args[1], target)+            expr.analyzed.line = expr.line+            expr.analyzed.accept(self)+        elif refers_to_fullname(expr.callee, 'builtins.reveal_type'):+            if not self.check_fixed_args(expr, 1, 'reveal_type'):+                return+            expr.analyzed = RevealExpr(kind=REVEAL_TYPE, expr=expr.args[0])+            expr.analyzed.line = expr.line+            expr.analyzed.column = expr.column+            expr.analyzed.accept(self)+        elif refers_to_fullname(expr.callee, 'builtins.reveal_locals'):+            # Store the local variable names into the RevealExpr for use in the+            # type checking pass+            local_nodes = []  # type: List[Var]+            if self.is_module_scope():+                # try to determine just the variable declarations in module scope+                # self.globals.values() contains SymbolTableNode's+                # Each SymbolTableNode has an attribute node that is nodes.Var+                # look for variable nodes that marked as is_inferred+                # Each symboltable node has a Var node as .node+                local_nodes = cast(+                    List[Var],+                    [+                        n.node for name, n in self.globals.items()+                        if getattr(n.node, 'is_inferred', False)+                    ]+                )+            elif self.is_class_scope():+                # type = None  # type: Optional[TypeInfo]+                if self.type is not None:+                    local_nodes = cast(List[Var], [st.node for st in self.type.names.values()])+            elif self.is_func_scope():+                # locals = None  # type: List[Optional[SymbolTable]]+                if self.locals is not None:+                    symbol_table = self.locals[-1]+                    if symbol_table is not None:+                        local_nodes = cast(List[Var], [st.node for st in symbol_table.values()])+            expr.analyzed = RevealExpr(kind=REVEAL_LOCALS, local_nodes=local_nodes)+            expr.analyzed.line = expr.line+            expr.analyzed.column = expr.column+            expr.analyzed.accept(self)+        elif refers_to_fullname(expr.callee, 'typing.Any'):+            # Special form Any(...) no longer supported.+            self.fail('Any(...) is no longer supported. Use cast(Any, ...) instead', expr)+        elif refers_to_fullname(expr.callee, 'typing._promote'):+            # Special form _promote(...).+            if not self.check_fixed_args(expr, 1, '_promote'):+                return+            # Translate first argument to an unanalyzed type.+            try:+                target = expr_to_unanalyzed_type(expr.args[0])+            except TypeTranslationError:+                self.fail('Argument 1 to _promote is not a type', expr)+                return+            expr.analyzed = PromoteExpr(target)+            expr.analyzed.line = expr.line+            expr.analyzed.accept(self)+        elif refers_to_fullname(expr.callee, 'builtins.dict'):+            expr.analyzed = self.translate_dict_call(expr)+        elif refers_to_fullname(expr.callee, 'builtins.divmod'):+            if not self.check_fixed_args(expr, 2, 'divmod'):+                return+            expr.analyzed = OpExpr('divmod', expr.args[0], expr.args[1])+            expr.analyzed.line = expr.line+            expr.analyzed.accept(self)+        else:+            # Normal call expression.+            for a in expr.args:+                a.accept(self)++            if (isinstance(expr.callee, MemberExpr) and+                    isinstance(expr.callee.expr, NameExpr) and+                    expr.callee.expr.name == '__all__' and+                    expr.callee.expr.kind == GDEF and+                    expr.callee.name in ('append', 'extend')):+                if expr.callee.name == 'append' and expr.args:+                    self.add_exports(expr.args[0])+                elif (expr.callee.name == 'extend' and expr.args and+                        isinstance(expr.args[0], (ListExpr, TupleExpr))):+                    self.add_exports(expr.args[0].items)++    def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]:+        """Translate 'dict(x=y, ...)' to {'x': y, ...}.++        For other variants of dict(...), return None.+        """+        if not call.args:+            return None+        if not all(kind == ARG_NAMED for kind in call.arg_kinds):+            # Must still accept those args.+            for a in call.args:+                a.accept(self)+            return None+        expr = DictExpr([(StrExpr(cast(str, key)), value)  # since they are all ARG_NAMED+                         for key, value in zip(call.arg_names, call.args)])+        expr.set_line(call)+        expr.accept(self)+        return expr++    def check_fixed_args(self, expr: CallExpr, numargs: int,+                         name: str) -> bool:+        """Verify that expr has specified number of positional args.++        Return True if the arguments are valid.+        """+        s = 's'+        if numargs == 1:+            s = ''+        if len(expr.args) != numargs:+            self.fail("'%s' expects %d argument%s" % (name, numargs, s),+                      expr)+            return False+        if expr.arg_kinds != [ARG_POS] * numargs:+            self.fail("'%s' must be called with %s positional argument%s" %+                      (name, numargs, s), expr)+            return False+        return True++    def visit_member_expr(self, expr: MemberExpr) -> None:+        base = expr.expr+        base.accept(self)+        # Bind references to module attributes.+        if isinstance(base, RefExpr) and base.kind == MODULE_REF:+            # This branch handles the case foo.bar where foo is a module.+            # In this case base.node is the module's MypyFile and we look up+            # bar in its namespace.  This must be done for all types of bar.+            file = cast(Optional[MypyFile], base.node)  # can't use isinstance due to issue #2999+            # TODO: Should we actually use this? Not sure if this makes a difference.+            # if file.fullname() == self.cur_mod_id:+            #     names = self.globals+            # else:+            #     names = file.names+            n = file.names.get(expr.name, None) if file is not None else None+            n = self.dereference_module_cross_ref(n)+            if n and not n.module_hidden:+                if not n:+                    return+                n = self.rebind_symbol_table_node(n)+                if n:+                    # TODO: What if None?+                    expr.kind = n.kind+                    expr.fullname = n.fullname+                    expr.node = n.node+            elif (file is not None and (file.is_stub or self.options.python_version >= (3, 7))+                    and '__getattr__' in file.names):+                # If there is a module-level __getattr__, then any attribute on the module is valid+                # per PEP 484.+                getattr_defn = file.names['__getattr__']+                if not getattr_defn:+                    typ = AnyType(TypeOfAny.from_error)  # type: Type+                elif isinstance(getattr_defn.node, (FuncDef, Var)):+                    if isinstance(getattr_defn.node.type, CallableType):+                        typ = getattr_defn.node.type.ret_type+                    else:+                        typ = AnyType(TypeOfAny.from_error)+                else:+                    typ = AnyType(TypeOfAny.from_error)+                expr.kind = MDEF+                expr.fullname = '{}.{}'.format(file.fullname(), expr.name)+                expr.node = Var(expr.name, type=typ)+            else:+                # We only catch some errors here; the rest will be+                # caught during type checking.+                #+                # This way we can report a larger number of errors in+                # one type checker run. If we reported errors here,+                # the build would terminate after semantic analysis+                # and we wouldn't be able to report any type errors.+                full_name = '%s.%s' % (file.fullname() if file is not None else None, expr.name)+                mod_name = " '%s'" % file.fullname() if file is not None else ''+                if full_name in obsolete_name_mapping:+                    self.fail("Module%s has no attribute %r (it's now called %r)" % (+                        mod_name, expr.name, obsolete_name_mapping[full_name]), expr)+        elif isinstance(base, RefExpr):+            # This branch handles the case C.bar (or cls.bar or self.bar inside+            # a classmethod/method), where C is a class and bar is a type+            # definition or a module resulting from `import bar` (or a module+            # assignment) inside class C. We look up bar in the class' TypeInfo+            # namespace.  This is done only when bar is a module or a type;+            # other things (e.g. methods) are handled by other code in+            # checkmember.+            type_info = None+            if isinstance(base.node, TypeInfo):+                # C.bar where C is a class+                type_info = base.node+            elif isinstance(base.node, Var) and self.type and self.function_stack:+                # check for self.bar or cls.bar in method/classmethod+                func_def = self.function_stack[-1]+                if not func_def.is_static and isinstance(func_def.type, CallableType):+                    formal_arg = func_def.type.argument_by_name(base.node.name())+                    if formal_arg and formal_arg.pos == 0:+                        type_info = self.type+            elif isinstance(base.node, TypeAlias) and base.node.no_args:+                if isinstance(base.node.target, Instance):+                    type_info = base.node.target.type++            if type_info:+                n = type_info.names.get(expr.name)+                if n is not None and (n.kind == MODULE_REF or isinstance(n.node, (TypeInfo,+                                                                                  TypeAlias))):+                    if not n:+                        return+                    expr.kind = n.kind+                    expr.fullname = n.fullname+                    expr.node = n.node++    def visit_op_expr(self, expr: OpExpr) -> None:+        expr.left.accept(self)++        if expr.op in ('and', 'or'):+            inferred = infer_condition_value(expr.left, self.options)+            if ((inferred == ALWAYS_FALSE and expr.op == 'and') or+                    (inferred == ALWAYS_TRUE and expr.op == 'or')):+                expr.right_unreachable = True+                return+            elif ((inferred == ALWAYS_TRUE and expr.op == 'and') or+                    (inferred == ALWAYS_FALSE and expr.op == 'or')):+                expr.right_always = True++        expr.right.accept(self)++    def visit_comparison_expr(self, expr: ComparisonExpr) -> None:+        for operand in expr.operands:+            operand.accept(self)++    def visit_unary_expr(self, expr: UnaryExpr) -> None:+        expr.expr.accept(self)++    def visit_index_expr(self, expr: IndexExpr) -> None:+        if expr.analyzed:+            return+        expr.base.accept(self)+        if (isinstance(expr.base, RefExpr)+                and isinstance(expr.base.node, TypeInfo)+                and not expr.base.node.is_generic()):+            expr.index.accept(self)+        elif (isinstance(expr.base, RefExpr) and isinstance(expr.base.node, TypeAlias) or+                refers_to_class_or_function(expr.base)):+            # Special form -- type application (either direct or via type aliasing).++            self.analyze_type_expr(expr.index)++            # Translate index to an unanalyzed type.+            types = []  # type: List[Type]+            if isinstance(expr.index, TupleExpr):+                items = expr.index.items+            else:+                items = [expr.index]+            for item in items:+                try:+                    typearg = expr_to_unanalyzed_type(item)+                except TypeTranslationError:+                    self.fail('Type expected within [...]', expr)+                    return+                # We always allow unbound type variables in IndexExpr, since we+                # may be analysing a type alias definition rvalue. The error will be+                # reported elsewhere if it is not the case.+                typearg = self.anal_type(typearg, allow_unbound_tvars=True)+                types.append(typearg)+            expr.analyzed = TypeApplication(expr.base, types)+            expr.analyzed.line = expr.line+            # Types list, dict, set are not subscriptable, prohibit this if+            # subscripted either via type alias...+            if isinstance(expr.base, RefExpr) and isinstance(expr.base.node, TypeAlias):+                alias = expr.base.node+                if isinstance(alias.target, Instance):+                    name = alias.target.type.fullname()+                    if (alias.no_args and  # this avoids bogus errors for already reported aliases+                            name in nongen_builtins and not alias.normalized):+                        self.fail(no_subscript_builtin_alias(name, propose_alt=False), expr)+            # ...or directly.+            else:+                n = self.lookup_type_node(expr.base)+                if n and n.fullname in nongen_builtins:+                    self.fail(no_subscript_builtin_alias(n.fullname, propose_alt=False), expr)+        else:+            expr.index.accept(self)++    def lookup_type_node(self, expr: Expression) -> Optional[SymbolTableNode]:+        try:+            t = expr_to_unanalyzed_type(expr)+        except TypeTranslationError:+            return None+        if isinstance(t, UnboundType):+            n = self.lookup_qualified(t.name, expr, suppress_errors=True)+            return n+        return None++    def visit_slice_expr(self, expr: SliceExpr) -> None:+        if expr.begin_index:+            expr.begin_index.accept(self)+        if expr.end_index:+            expr.end_index.accept(self)+        if expr.stride:+            expr.stride.accept(self)++    def visit_cast_expr(self, expr: CastExpr) -> None:+        expr.expr.accept(self)+        expr.type = self.anal_type(expr.type)++    def visit_reveal_expr(self, expr: RevealExpr) -> None:+        if expr.kind == REVEAL_TYPE:+            if expr.expr is not None:+                expr.expr.accept(self)+        else:+            # Reveal locals doesn't have an inner expression, there's no+            # need to traverse inside it+            pass++    def visit_type_application(self, expr: TypeApplication) -> None:+        expr.expr.accept(self)+        for i in range(len(expr.types)):+            expr.types[i] = self.anal_type(expr.types[i])++    def visit_list_comprehension(self, expr: ListComprehension) -> None:+        expr.generator.accept(self)++    def visit_set_comprehension(self, expr: SetComprehension) -> None:+        expr.generator.accept(self)++    def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> None:+        self.enter()+        self.analyze_comp_for(expr)+        expr.key.accept(self)+        expr.value.accept(self)+        self.leave()+        self.analyze_comp_for_2(expr)++    def visit_generator_expr(self, expr: GeneratorExpr) -> None:+        self.enter()+        self.analyze_comp_for(expr)+        expr.left_expr.accept(self)+        self.leave()+        self.analyze_comp_for_2(expr)++    def analyze_comp_for(self, expr: Union[GeneratorExpr,+                                           DictionaryComprehension]) -> None:+        """Analyses the 'comp_for' part of comprehensions (part 1).++        That is the part after 'for' in (x for x in l if p). This analyzes+        variables and conditions which are analyzed in a local scope.+        """+        for i, (index, sequence, conditions) in enumerate(zip(expr.indices,+                                                              expr.sequences,+                                                              expr.condlists)):+            if i > 0:+                sequence.accept(self)+            # Bind index variables.+            self.analyze_lvalue(index)+            for cond in conditions:+                cond.accept(self)++    def analyze_comp_for_2(self, expr: Union[GeneratorExpr,+                                             DictionaryComprehension]) -> None:+        """Analyses the 'comp_for' part of comprehensions (part 2).++        That is the part after 'for' in (x for x in l if p). This analyzes+        the 'l' part which is analyzed in the surrounding scope.+        """+        expr.sequences[0].accept(self)++    def visit_lambda_expr(self, expr: LambdaExpr) -> None:+        self.analyze_function(expr)++    def visit_conditional_expr(self, expr: ConditionalExpr) -> None:+        expr.if_expr.accept(self)+        expr.cond.accept(self)+        expr.else_expr.accept(self)++    def visit_backquote_expr(self, expr: BackquoteExpr) -> None:+        expr.expr.accept(self)++    def visit__promote_expr(self, expr: PromoteExpr) -> None:+        expr.type = self.anal_type(expr.type)++    def visit_yield_expr(self, expr: YieldExpr) -> None:+        if not self.is_func_scope():+            self.fail("'yield' outside function", expr, True, blocker=True)+        else:+            if self.function_stack[-1].is_coroutine:+                if self.options.python_version < (3, 6):+                    self.fail("'yield' in async function", expr, True, blocker=True)+                else:+                    self.function_stack[-1].is_generator = True+                    self.function_stack[-1].is_async_generator = True+            else:+                self.function_stack[-1].is_generator = True+        if expr.expr:+            expr.expr.accept(self)++    def visit_await_expr(self, expr: AwaitExpr) -> None:+        if not self.is_func_scope():+            self.fail("'await' outside function", expr)+        elif not self.function_stack[-1].is_coroutine:+            self.fail("'await' outside coroutine ('async def')", expr)+        expr.expr.accept(self)++    #+    # Helpers+    #++    @contextmanager+    def tvar_scope_frame(self, frame: TypeVarScope) -> Iterator[None]:+        old_scope = self.tvar_scope+        self.tvar_scope = frame+        yield+        self.tvar_scope = old_scope++    def lookup(self, name: str, ctx: Context,+               suppress_errors: bool = False) -> Optional[SymbolTableNode]:+        """Look up an unqualified name in all active namespaces."""+        implicit_name = False+        # 1a. Name declared using 'global x' takes precedence+        if name in self.global_decls[-1]:+            if name in self.globals:+                return self.globals[name]+            if not suppress_errors:+                self.name_not_defined(name, ctx)+            return None+        # 1b. Name declared using 'nonlocal x' takes precedence+        if name in self.nonlocal_decls[-1]:+            for table in reversed(self.locals[:-1]):+                if table is not None and name in table:+                    return table[name]+            else:+                if not suppress_errors:+                    self.name_not_defined(name, ctx)+                return None+        # 2. Class attributes (if within class definition)+        if self.type and not self.is_func_scope() and name in self.type.names:+            node = self.type.names[name]+            if not node.implicit:+                return node+            implicit_name = True+            implicit_node = node+        # 3. Local (function) scopes+        for table in reversed(self.locals):+            if table is not None and name in table:+                return table[name]+        # 4. Current file global scope+        if name in self.globals:+            return self.globals[name]+        # 5. Builtins+        b = self.globals.get('__builtins__', None)+        if b:+            assert isinstance(b.node, MypyFile)+            table = b.node.names+            if name in table:+                if name[0] == "_" and name[1] != "_":+                    if not suppress_errors:+                        self.name_not_defined(name, ctx)+                    return None+                node = table[name]+                return node+        # Give up.+        if not implicit_name and not suppress_errors:+            self.name_not_defined(name, ctx)+            self.check_for_obsolete_short_name(name, ctx)+        else:+            if implicit_name:+                return implicit_node+        return None++    def check_for_obsolete_short_name(self, name: str, ctx: Context) -> None:+        matches = [obsolete_name+                   for obsolete_name in obsolete_name_mapping+                   if obsolete_name.rsplit('.', 1)[-1] == name]+        if len(matches) == 1:+            self.note("(Did you mean '{}'?)".format(obsolete_name_mapping[matches[0]]), ctx)++    def lookup_qualified(self, name: str, ctx: Context,+                         suppress_errors: bool = False) -> Optional[SymbolTableNode]:+        if '.' not in name:+            return self.lookup(name, ctx, suppress_errors=suppress_errors)+        else:+            parts = name.split('.')+            n = self.lookup(parts[0], ctx, suppress_errors=suppress_errors)+            if n:+                for i in range(1, len(parts)):+                    if isinstance(n.node, TypeInfo):+                        if not n.node.mro:+                            # We haven't yet analyzed the class `n.node`.  Fall back to direct+                            # lookup in the names declared directly under it, without its base+                            # classes.  This can happen when we have a forward reference to a+                            # nested class, and the reference is bound before the outer class+                            # has been fully semantically analyzed.+                            #+                            # A better approach would be to introduce a new analysis pass or+                            # to move things around between passes, but this unblocks a common+                            # use case even though this is a little limited in case there is+                            # inheritance involved.+                            result = n.node.names.get(parts[i])+                        else:+                            result = n.node.get(parts[i])+                        n = result+                    elif isinstance(n.node, MypyFile):+                        names = n.node.names+                        # Rebind potential references to old version of current module in+                        # fine-grained incremental mode.+                        #+                        # TODO: Do this for all modules in the set of modified files.+                        if n.node.fullname() == self.cur_mod_id:+                            names = self.globals+                        n = names.get(parts[i], None)+                        if n and isinstance(n.node, ImportedName):+                            n = self.dereference_module_cross_ref(n)+                        elif not n and '__getattr__' in names:+                            gvar = self.create_getattr_var(names['__getattr__'],+                                                           parts[i], parts[i])+                            if gvar:+                                names[name] = gvar+                                n = gvar+                    # TODO: What if node is Var or FuncDef?+                    # Currently, missing these cases results in controversial behavior, when+                    # lookup_qualified(x.y.z) returns Var(x).+                    if not n:+                        if not suppress_errors:+                            self.name_not_defined(name, ctx)+                        break+                if n:+                    if n and n.module_hidden:+                        self.name_not_defined(name, ctx)+            if n and not n.module_hidden:+                n = self.rebind_symbol_table_node(n)+                return n+            return None++    def create_getattr_var(self, getattr_defn: SymbolTableNode,+                           name: str, fullname: str) -> Optional[SymbolTableNode]:+        """Create a dummy global symbol using __getattr__ return type.++        If not possible, return None.+        """+        if isinstance(getattr_defn.node, (FuncDef, Var)):+            if isinstance(getattr_defn.node.type, CallableType):+                typ = getattr_defn.node.type.ret_type+            else:+                typ = AnyType(TypeOfAny.from_error)+            v = Var(name, type=typ)+            v._fullname = fullname+            return SymbolTableNode(GDEF, v)+        return None++    def rebind_symbol_table_node(self, n: SymbolTableNode) -> Optional[SymbolTableNode]:+        """If node refers to old version of module, return reference to new version.++        If the reference is removed in the new version, return None.+        """+        # TODO: Handle type variables and other sorts of references+        if isinstance(n.node, (FuncDef, OverloadedFuncDef, TypeInfo, Var, TypeAlias)):+            # TODO: Why is it possible for fullname() to be None, even though it's not+            #   annotated as Optional[str]?+            # TODO: Do this for all modules in the set of modified files+            # TODO: This doesn't work for things nested within classes+            if n.node.fullname() and get_prefix(n.node.fullname()) == self.cur_mod_id:+                # This is an indirect reference to a name defined in the current module.+                # Rebind it.+                return self.globals.get(n.node.name())+        # No need to rebind.+        return n++    def builtin_type(self, fully_qualified_name: str) -> Instance:+        sym = self.lookup_fully_qualified(fully_qualified_name)+        node = sym.node+        assert isinstance(node, TypeInfo)+        return Instance(node, [AnyType(TypeOfAny.special_form)] * len(node.defn.type_vars))++    def add_builtin_aliases(self, tree: MypyFile) -> None:+        """Add builtin type aliases to typing module.++        For historical reasons, the aliases like `List = list` are not defined+        in typeshed stubs for typing module. Instead we need to manually add the+        corresponding nodes on the fly. We explicitly mark these aliases as normalized,+        so that a user can write `typing.List[int]`.+        """+        assert tree.fullname() == 'typing'+        for alias, target_name in type_aliases.items():+            name = alias.split('.')[-1]+            n = self.lookup_fully_qualified_or_none(target_name)+            if n:+                target = self.named_type_or_none(target_name, [])+                assert target is not None+                alias_node = TypeAlias(target, alias, line=-1, column=-1,  # there is no context+                                       no_args=True, normalized=True)+                tree.names[name] = SymbolTableNode(GDEF, alias_node)+            else:+                # Built-in target not defined, remove the original fake+                # definition to trigger a better error message.+                tree.names.pop(name, None)++    def lookup_fully_qualified(self, name: str) -> SymbolTableNode:+        """Lookup a fully qualified name.++        Assume that the name is defined. This happens in the global namespace -- the local+        module namespace is ignored.+        """+        parts = name.split('.')+        n = self.modules[parts[0]]+        for i in range(1, len(parts) - 1):+            next_sym = n.names[parts[i]]+            assert isinstance(next_sym.node, MypyFile)+            n = next_sym.node+        return n.names[parts[-1]]++    def lookup_fully_qualified_or_none(self, fullname: str) -> Optional[SymbolTableNode]:+        """Lookup a fully qualified name that refers to a module-level definition.++        Don't assume that the name is defined. This happens in the global namespace --+        the local module namespace is ignored. This does not dereference indirect+        refs.++        Note that this can't be used for names nested in class namespaces.+        """+        assert '.' in fullname+        module, name = fullname.rsplit('.', maxsplit=1)+        if module not in self.modules:+            return None+        filenode = self.modules[module]+        return filenode.names.get(name)++    def qualified_name(self, n: str) -> str:+        if self.type is not None:+            base = self.type._fullname+        else:+            base = self.cur_mod_id+        return base + '.' + n++    def enter(self) -> None:+        self.locals.append(SymbolTable())+        self.global_decls.append(set())+        self.nonlocal_decls.append(set())+        # -1 since entering block will increment this to 0.+        self.block_depth.append(-1)++    def leave(self) -> None:+        self.locals.pop()+        self.global_decls.pop()+        self.nonlocal_decls.pop()+        self.block_depth.pop()++    def is_func_scope(self) -> bool:+        return self.locals[-1] is not None++    def is_nested_within_func_scope(self) -> bool:+        """Are we underneath a function scope, even if we are in a nested class also"""+        return any(l is not None for l in self.locals)++    def is_class_scope(self) -> bool:+        return self.type is not None and not self.is_func_scope()++    def is_module_scope(self) -> bool:+        return not (self.is_class_scope() or self.is_func_scope())++    def add_symbol(self, name: str, node: SymbolTableNode,+                   context: Context) -> None:+        # NOTE: This logic mostly parallels SemanticAnalyzerPass1.add_symbol. If you change+        #     this, you may have to change the other method as well.+        if self.is_func_scope():+            assert self.locals[-1] is not None+            if name in self.locals[-1]:+                # Flag redefinition unless this is a reimport of a module.+                if not (node.kind == MODULE_REF and+                        self.locals[-1][name].node == node.node):+                    self.name_already_defined(name, context, self.locals[-1][name])+            self.locals[-1][name] = node+        elif self.type:+            self.type.names[name] = node+        else:+            existing = self.globals.get(name)+            if (existing+                    and (not isinstance(node.node, MypyFile) or existing.node != node.node)+                    and existing.kind != UNBOUND_IMPORTED+                    and not isinstance(existing.node, ImportedName)):+                # Modules can be imported multiple times to support import+                # of multiple submodules of a package (e.g. a.x and a.y).+                ok = False+                # Only report an error if the symbol collision provides a different type.+                if existing.type and node.type and is_same_type(existing.type, node.type):+                    ok = True+                if not ok:+                    self.name_already_defined(name, context, existing)+            self.globals[name] = node++    def add_local(self, node: Union[Var, FuncDef, OverloadedFuncDef], ctx: Context) -> None:+        assert self.locals[-1] is not None, "Should not add locals outside a function"+        name = node.name()+        if name in self.locals[-1]:+            self.name_already_defined(name, ctx, self.locals[-1][name])+        node._fullname = name+        self.locals[-1][name] = SymbolTableNode(LDEF, node)++    def add_exports(self, exp_or_exps: Union[Iterable[Expression], Expression]) -> None:+        exps = [exp_or_exps] if isinstance(exp_or_exps, Expression) else exp_or_exps+        for exp in exps:+            if isinstance(exp, StrExpr):+                self.all_exports.add(exp.value)++    def check_no_global(self, n: str, ctx: Context,+                        is_overloaded_func: bool = False) -> None:+        if n in self.globals:+            prev_is_overloaded = isinstance(self.globals[n], OverloadedFuncDef)+            if is_overloaded_func and prev_is_overloaded:+                self.fail("Nonconsecutive overload {} found".format(n), ctx)+            elif prev_is_overloaded:+                self.fail("Definition of '{}' missing 'overload'".format(n), ctx)+            else:+                self.name_already_defined(n, ctx, self.globals[n])++    def name_not_defined(self, name: str, ctx: Context) -> None:+        message = "Name '{}' is not defined".format(name)+        extra = self.undefined_name_extra_info(name)+        if extra:+            message += ' {}'.format(extra)+        self.fail(message, ctx)+        if 'builtins.{}'.format(name) in SUGGESTED_TEST_FIXTURES:+            # The user probably has a missing definition in a test fixture. Let's verify.+            fullname = 'builtins.{}'.format(name)+            if self.lookup_fully_qualified_or_none(fullname) is None:+                # Yes. Generate a helpful note.+                self.add_fixture_note(fullname, ctx)++    def name_already_defined(self, name: str, ctx: Context,+                    original_ctx: Optional[Union[SymbolTableNode, SymbolNode]] = None) -> None:+        if isinstance(original_ctx, SymbolTableNode):+            node = original_ctx.node+        elif isinstance(original_ctx, SymbolNode):+            node = original_ctx++        if isinstance(original_ctx, SymbolTableNode) and original_ctx.kind == MODULE_REF:+            # Since this is an import, original_ctx.node points to the module definition.+            # Therefore its line number is always 1, which is not useful for this+            # error message.+            extra_msg = ' (by an import)'+        elif node and node.line != -1:+            extra_msg = ' on line {}'.format(node.line)+        else:+            extra_msg = ' (possibly by an import)'+        self.fail("Name '{}' already defined{}".format(name, extra_msg), ctx)++    def fail(self, msg: str, ctx: Context, serious: bool = False, *,+             blocker: bool = False) -> None:+        if (not serious and+                not self.options.check_untyped_defs and+                self.function_stack and+                self.function_stack[-1].is_dynamic()):+            return+        # In case it's a bug and we don't really have context+        assert ctx is not None, msg+        self.errors.report(ctx.get_line(), ctx.get_column(), msg, blocker=blocker)++    def fail_blocker(self, msg: str, ctx: Context) -> None:+        self.fail(msg, ctx, blocker=True)++    def note(self, msg: str, ctx: Context) -> None:+        if (not self.options.check_untyped_defs and+                self.function_stack and+                self.function_stack[-1].is_dynamic()):+            return+        self.errors.report(ctx.get_line(), ctx.get_column(), msg, severity='note')++    def undefined_name_extra_info(self, fullname: str) -> Optional[str]:+        if fullname in obsolete_name_mapping:+            return "(it's now called '{}')".format(obsolete_name_mapping[fullname])+        else:+            return None++    def accept(self, node: Node) -> None:+        try:+            node.accept(self)+        except Exception as err:+            report_internal_error(err, self.errors.file, node.line, self.errors, self.options)++    def analyze_type_expr(self, expr: Expression) -> None:+        # There are certain expressions that mypy does not need to semantically analyze,+        # since they analyzed solely as type. (For example, indexes in type alias definitions+        # and base classes in class defs). External consumers of the mypy AST may need+        # them semantically analyzed, however, if they need to treat it as an expression+        # and not a type. (Which is to say, mypyc needs to do this.) Do the analysis+        # in a fresh tvar scope in order to suppress any errors about using type variables.+        with self.tvar_scope_frame(TypeVarScope()):+            expr.accept(self)++    def lookup_current_scope(self, name: str) -> Optional[SymbolTableNode]:+        if self.locals[-1] is not None:+            return self.locals[-1].get(name)+        elif self.type is not None:+            return self.type.names.get(name)+        else:+            return self.globals.get(name)++    def schedule_patch(self, priority: int, patch: Callable[[], None]) -> None:+        self.patches.append((priority, patch))++    def add_symbol_table_node(self, name: str, stnode: SymbolTableNode) -> None:+        """Add node to global symbol table (or to nearest class if there is one)."""+        # TODO: Adding to the nearest class is ad hoc.+        if self.type:+            self.type.names[name] = stnode+        else:+            self.globals[name] = stnode+++def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike:+    if isinstance(sig, CallableType):+        if len(sig.arg_types) == 0:+            return sig+        return sig.copy_modified(arg_types=[new] + sig.arg_types[1:])+    elif isinstance(sig, Overloaded):+        return Overloaded([cast(CallableType, replace_implicit_first_type(i, new))+                           for i in sig.items()])+    else:+        assert False+++def refers_to_fullname(node: Expression, fullname: str) -> bool:+    """Is node a name or member expression with the given full name?"""+    if not isinstance(node, RefExpr):+        return False+    return (node.fullname == fullname or+            isinstance(node.node, TypeAlias) and isinstance(node.node.target, Instance)+            and node.node.target.type.fullname() == fullname)+++def refers_to_class_or_function(node: Expression) -> bool:+    """Does semantically analyzed node refer to a class?"""+    return (isinstance(node, RefExpr) and+            isinstance(node.node, (TypeInfo, FuncDef, OverloadedFuncDef)))+++def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> None:+    try:+        calculate_mro(defn.info)+    except MroError:+        fail("Cannot determine consistent method resolution order "+             '(MRO) for "%s"' % defn.name, defn)+        defn.info.mro = []+++def calculate_mro(info: TypeInfo) -> None:+    """Calculate and set mro (method resolution order).++    Raise MroError if cannot determine mro.+    """+    mro = linearize_hierarchy(info)+    assert mro, "Could not produce a MRO at all for %s" % (info,)+    info.mro = mro+    # The property of falling back to Any is inherited.+    info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in info.mro)+    TypeState.reset_all_subtype_caches_for(info)+++class MroError(Exception):+    """Raised if a consistent mro cannot be determined for a class."""+++def linearize_hierarchy(info: TypeInfo) -> List[TypeInfo]:+    # TODO describe+    if info.mro:+        return info.mro+    bases = info.direct_base_classes()+    lin_bases = []+    for base in bases:+        assert base is not None, "Cannot linearize bases for %s %s" % (info.fullname(), bases)+        lin_bases.append(linearize_hierarchy(base))+    lin_bases.append(bases)+    return [info] + merge(lin_bases)+++def merge(seqs: List[List[TypeInfo]]) -> List[TypeInfo]:+    seqs = [s[:] for s in seqs]+    result = []  # type: List[TypeInfo]+    while True:+        seqs = [s for s in seqs if s]+        if not seqs:+            return result+        for seq in seqs:+            head = seq[0]+            if not [s for s in seqs if head in s[1:]]:+                break+        else:+            raise MroError()+        result.append(head)+        for s in seqs:+            if s[0] is head:+                del s[0]+++def find_duplicate(list: List[T]) -> Optional[T]:+    """If the list has duplicates, return one of the duplicates.++    Otherwise, return None.+    """+    for i in range(1, len(list)):+        if list[i] in list[:i]:+            return list[i]+    return None+++def remove_imported_names_from_symtable(names: SymbolTable,+                                        module: str) -> None:+    """Remove all imported names from the symbol table of a module."""+    removed = []  # type: List[str]+    for name, node in names.items():+        if node.node is None:+            continue+        fullname = node.node.fullname()+        prefix = fullname[:fullname.rfind('.')]+        if prefix != module:+            removed.append(name)+    for name in removed:+        del names[name]+++def infer_reachability_of_if_statement(s: IfStmt, options: Options) -> None:+    for i in range(len(s.expr)):+        result = infer_condition_value(s.expr[i], options)+        if result in (ALWAYS_FALSE, MYPY_FALSE):+            # The condition is considered always false, so we skip the if/elif body.+            mark_block_unreachable(s.body[i])+        elif result in (ALWAYS_TRUE, MYPY_TRUE):+            # This condition is considered always true, so all of the remaining+            # elif/else bodies should not be checked.+            if result == MYPY_TRUE:+                # This condition is false at runtime; this will affect+                # import priorities.+                mark_block_mypy_only(s.body[i])+            for body in s.body[i + 1:]:+                mark_block_unreachable(body)++            # Make sure else body always exists and is marked as+            # unreachable so the type checker always knows that+            # all control flow paths will flow through the if+            # statement body.+            if not s.else_body:+                s.else_body = Block([])+            mark_block_unreachable(s.else_body)+            break+++def infer_condition_value(expr: Expression, options: Options) -> int:+    """Infer whether the given condition is always true/false.++    Return ALWAYS_TRUE if always true, ALWAYS_FALSE if always false,+    MYPY_TRUE if true under mypy and false at runtime, MYPY_FALSE if+    false under mypy and true at runtime, else TRUTH_VALUE_UNKNOWN.+    """+    pyversion = options.python_version+    name = ''+    negated = False+    alias = expr+    if isinstance(alias, UnaryExpr):+        if alias.op == 'not':+            expr = alias.expr+            negated = True+    result = TRUTH_VALUE_UNKNOWN+    if isinstance(expr, NameExpr):+        name = expr.name+    elif isinstance(expr, MemberExpr):+        name = expr.name+    elif isinstance(expr, OpExpr) and expr.op in ('and', 'or'):+        left = infer_condition_value(expr.left, options)+        if ((left == ALWAYS_TRUE and expr.op == 'and') or+                (left == ALWAYS_FALSE and expr.op == 'or')):+            # Either `True and <other>` or `False or <other>`: the result will+            # always be the right-hand-side.+            return infer_condition_value(expr.right, options)+        else:+            # The result will always be the left-hand-side (e.g. ALWAYS_* or+            # TRUTH_VALUE_UNKNOWN).+            return left+    else:+        result = consider_sys_version_info(expr, pyversion)+        if result == TRUTH_VALUE_UNKNOWN:+            result = consider_sys_platform(expr, options.platform)+    if result == TRUTH_VALUE_UNKNOWN:+        if name == 'PY2':+            result = ALWAYS_TRUE if pyversion[0] == 2 else ALWAYS_FALSE+        elif name == 'PY3':+            result = ALWAYS_TRUE if pyversion[0] == 3 else ALWAYS_FALSE+        elif name == 'MYPY' or name == 'TYPE_CHECKING':+            result = MYPY_TRUE+        elif name in options.always_true:+            result = MYPY_TRUE+        elif name in options.always_false:+            result = MYPY_FALSE+    if negated:+        result = inverted_truth_mapping[result]+    return result+++def consider_sys_version_info(expr: Expression, pyversion: Tuple[int, ...]) -> int:+    """Consider whether expr is a comparison involving sys.version_info.++    Return ALWAYS_TRUE, ALWAYS_FALSE, or TRUTH_VALUE_UNKNOWN.+    """+    # Cases supported:+    # - sys.version_info[<int>] <compare_op> <int>+    # - sys.version_info[:<int>] <compare_op> <tuple_of_n_ints>+    # - sys.version_info <compare_op> <tuple_of_1_or_2_ints>+    #   (in this case <compare_op> must be >, >=, <, <=, but cannot be ==, !=)+    if not isinstance(expr, ComparisonExpr):+        return TRUTH_VALUE_UNKNOWN+    # Let's not yet support chained comparisons.+    if len(expr.operators) > 1:+        return TRUTH_VALUE_UNKNOWN+    op = expr.operators[0]+    if op not in ('==', '!=', '<=', '>=', '<', '>'):+        return TRUTH_VALUE_UNKNOWN+    thing = contains_int_or_tuple_of_ints(expr.operands[1])+    if thing is None:+        return TRUTH_VALUE_UNKNOWN+    index = contains_sys_version_info(expr.operands[0])+    if isinstance(index, int) and isinstance(thing, int):+        # sys.version_info[i] <compare_op> k+        if 0 <= index <= 1:+            return fixed_comparison(pyversion[index], op, thing)+        else:+            return TRUTH_VALUE_UNKNOWN+    elif isinstance(index, tuple) and isinstance(thing, tuple):+        lo, hi = index+        if lo is None:+            lo = 0+        if hi is None:+            hi = 2+        if 0 <= lo < hi <= 2:+            val = pyversion[lo:hi]+            if len(val) == len(thing) or len(val) > len(thing) and op not in ('==', '!='):+                return fixed_comparison(val, op, thing)+    return TRUTH_VALUE_UNKNOWN+++def consider_sys_platform(expr: Expression, platform: str) -> int:+    """Consider whether expr is a comparison involving sys.platform.++    Return ALWAYS_TRUE, ALWAYS_FALSE, or TRUTH_VALUE_UNKNOWN.+    """+    # Cases supported:+    # - sys.platform == 'posix'+    # - sys.platform != 'win32'+    # - sys.platform.startswith('win')+    if isinstance(expr, ComparisonExpr):+        # Let's not yet support chained comparisons.+        if len(expr.operators) > 1:+            return TRUTH_VALUE_UNKNOWN+        op = expr.operators[0]+        if op not in ('==', '!='):+            return TRUTH_VALUE_UNKNOWN+        if not is_sys_attr(expr.operands[0], 'platform'):+            return TRUTH_VALUE_UNKNOWN+        right = expr.operands[1]+        if not isinstance(right, (StrExpr, UnicodeExpr)):+            return TRUTH_VALUE_UNKNOWN+        return fixed_comparison(platform, op, right.value)+    elif isinstance(expr, CallExpr):+        if not isinstance(expr.callee, MemberExpr):+            return TRUTH_VALUE_UNKNOWN+        if len(expr.args) != 1 or not isinstance(expr.args[0], (StrExpr, UnicodeExpr)):+            return TRUTH_VALUE_UNKNOWN+        if not is_sys_attr(expr.callee.expr, 'platform'):+            return TRUTH_VALUE_UNKNOWN+        if expr.callee.name != 'startswith':+            return TRUTH_VALUE_UNKNOWN+        if platform.startswith(expr.args[0].value):+            return ALWAYS_TRUE+        else:+            return ALWAYS_FALSE+    else:+        return TRUTH_VALUE_UNKNOWN+++Targ = TypeVar('Targ', int, str, Tuple[int, ...])+++def fixed_comparison(left: Targ, op: str, right: Targ) -> int:+    rmap = {False: ALWAYS_FALSE, True: ALWAYS_TRUE}+    if op == '==':+        return rmap[left == right]+    if op == '!=':+        return rmap[left != right]+    if op == '<=':+        return rmap[left <= right]+    if op == '>=':+        return rmap[left >= right]+    if op == '<':+        return rmap[left < right]+    if op == '>':+        return rmap[left > right]+    return TRUTH_VALUE_UNKNOWN+++def contains_int_or_tuple_of_ints(expr: Expression+                                  ) -> Union[None, int, Tuple[int], Tuple[int, ...]]:+    if isinstance(expr, IntExpr):+        return expr.value+    if isinstance(expr, TupleExpr):+        if literal(expr) == LITERAL_YES:+            thing = []+            for x in expr.items:+                if not isinstance(x, IntExpr):+                    return None+                thing.append(x.value)+            return tuple(thing)+    return None+++def contains_sys_version_info(expr: Expression+                              ) -> Union[None, int, Tuple[Optional[int], Optional[int]]]:+    if is_sys_attr(expr, 'version_info'):+        return (None, None)  # Same as sys.version_info[:]+    if isinstance(expr, IndexExpr) and is_sys_attr(expr.base, 'version_info'):+        index = expr.index+        if isinstance(index, IntExpr):+            return index.value+        if isinstance(index, SliceExpr):+            if index.stride is not None:+                if not isinstance(index.stride, IntExpr) or index.stride.value != 1:+                    return None+            begin = end = None+            if index.begin_index is not None:+                if not isinstance(index.begin_index, IntExpr):+                    return None+                begin = index.begin_index.value+            if index.end_index is not None:+                if not isinstance(index.end_index, IntExpr):+                    return None+                end = index.end_index.value+            return (begin, end)+    return None+++def is_sys_attr(expr: Expression, name: str) -> bool:+    # TODO: This currently doesn't work with code like this:+    # - import sys as _sys+    # - from sys import version_info+    if isinstance(expr, MemberExpr) and expr.name == name:+        if isinstance(expr.expr, NameExpr) and expr.expr.name == 'sys':+            # TODO: Guard against a local named sys, etc.+            # (Though later passes will still do most checking.)+            return True+    return False+++def mark_block_unreachable(block: Block) -> None:+    block.is_unreachable = True+    block.accept(MarkImportsUnreachableVisitor())+++class MarkImportsUnreachableVisitor(TraverserVisitor):+    """Visitor that flags all imports nested within a node as unreachable."""++    def visit_import(self, node: Import) -> None:+        node.is_unreachable = True++    def visit_import_from(self, node: ImportFrom) -> None:+        node.is_unreachable = True++    def visit_import_all(self, node: ImportAll) -> None:+        node.is_unreachable = True+++def mark_block_mypy_only(block: Block) -> None:+    block.accept(MarkImportsMypyOnlyVisitor())+++class MarkImportsMypyOnlyVisitor(TraverserVisitor):+    """Visitor that sets is_mypy_only (which affects priority)."""++    def visit_import(self, node: Import) -> None:+        node.is_mypy_only = True++    def visit_import_from(self, node: ImportFrom) -> None:+        node.is_mypy_only = True++    def visit_import_all(self, node: ImportAll) -> None:+        node.is_mypy_only = True+++def make_any_non_explicit(t: Type) -> Type:+    """Replace all Any types within in with Any that has attribute 'explicit' set to False"""+    return t.accept(MakeAnyNonExplicit())+++class MakeAnyNonExplicit(TypeTranslator):+    def visit_any(self, t: AnyType) -> Type:+        if t.type_of_any == TypeOfAny.explicit:+            return t.copy_modified(TypeOfAny.special_form)+        return t+++def apply_semantic_analyzer_patches(patches: List[Tuple[int, Callable[[], None]]]) -> None:+    """Call patch callbacks in the right order.++    This should happen after semantic analyzer pass 3.+    """+    patches_by_priority = sorted(patches, key=lambda x: x[0])+    for priority, patch_func in patches_by_priority:+        patch_func()
+ test/files/numpy.py view
@@ -0,0 +1,8077 @@+"""+numpy.ma : a package to handle missing or invalid values.++This package was initially written for numarray by Paul F. Dubois+at Lawrence Livermore National Laboratory.+In 2006, the package was completely rewritten by Pierre Gerard-Marchant+(University of Georgia) to make the MaskedArray class a subclass of ndarray,+and to improve support of structured arrays.+++Copyright 1999, 2000, 2001 Regents of the University of California.+Released for unlimited redistribution.++* Adapted for numpy_core 2005 by Travis Oliphant and (mainly) Paul Dubois.+* Subclassing of the base `ndarray` 2006 by Pierre Gerard-Marchant+  (pgmdevlist_AT_gmail_DOT_com)+* Improvements suggested by Reggie Dugard (reggie_AT_merfinllc_DOT_com)++.. moduleauthor:: Pierre Gerard-Marchant++"""+# pylint: disable-msg=E1002+from __future__ import division, absolute_import, print_function++import sys+import operator+import warnings+import textwrap+import re+from functools import reduce++if sys.version_info[0] >= 3:+    import builtins+else:+    import __builtin__ as builtins++import numpy as np+import numpy.core.umath as umath+import numpy.core.numerictypes as ntypes+from numpy import ndarray, amax, amin, iscomplexobj, bool_, _NoValue+from numpy import array as narray+from numpy.lib.function_base import angle+from numpy.compat import (+    getargspec, formatargspec, long, basestring, unicode, bytes+    )+from numpy import expand_dims+from numpy.core.multiarray import normalize_axis_index+from numpy.core.numeric import normalize_axis_tuple+++if sys.version_info[0] >= 3:+    import pickle+else:+    import cPickle as pickle++__all__ = [+    'MAError', 'MaskError', 'MaskType', 'MaskedArray', 'abs', 'absolute',+    'add', 'all', 'allclose', 'allequal', 'alltrue', 'amax', 'amin',+    'angle', 'anom', 'anomalies', 'any', 'append', 'arange', 'arccos',+    'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh',+    'argmax', 'argmin', 'argsort', 'around', 'array', 'asanyarray',+    'asarray', 'bitwise_and', 'bitwise_or', 'bitwise_xor', 'bool_', 'ceil',+    'choose', 'clip', 'common_fill_value', 'compress', 'compressed',+    'concatenate', 'conjugate', 'convolve', 'copy', 'correlate', 'cos', 'cosh',+    'count', 'cumprod', 'cumsum', 'default_fill_value', 'diag', 'diagonal',+    'diff', 'divide', 'dump', 'dumps', 'empty', 'empty_like', 'equal', 'exp',+    'expand_dims', 'fabs', 'filled', 'fix_invalid', 'flatten_mask',+    'flatten_structured_array', 'floor', 'floor_divide', 'fmod',+    'frombuffer', 'fromflex', 'fromfunction', 'getdata', 'getmask',+    'getmaskarray', 'greater', 'greater_equal', 'harden_mask', 'hypot',+    'identity', 'ids', 'indices', 'inner', 'innerproduct', 'isMA',+    'isMaskedArray', 'is_mask', 'is_masked', 'isarray', 'left_shift',+    'less', 'less_equal', 'load', 'loads', 'log', 'log10', 'log2',+    'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'make_mask',+    'make_mask_descr', 'make_mask_none', 'mask_or', 'masked',+    'masked_array', 'masked_equal', 'masked_greater',+    'masked_greater_equal', 'masked_inside', 'masked_invalid',+    'masked_less', 'masked_less_equal', 'masked_not_equal',+    'masked_object', 'masked_outside', 'masked_print_option',+    'masked_singleton', 'masked_values', 'masked_where', 'max', 'maximum',+    'maximum_fill_value', 'mean', 'min', 'minimum', 'minimum_fill_value',+    'mod', 'multiply', 'mvoid', 'ndim', 'negative', 'nomask', 'nonzero',+    'not_equal', 'ones', 'outer', 'outerproduct', 'power', 'prod',+    'product', 'ptp', 'put', 'putmask', 'rank', 'ravel', 'remainder',+    'repeat', 'reshape', 'resize', 'right_shift', 'round', 'round_',+    'set_fill_value', 'shape', 'sin', 'sinh', 'size', 'soften_mask',+    'sometrue', 'sort', 'sqrt', 'squeeze', 'std', 'subtract', 'sum',+    'swapaxes', 'take', 'tan', 'tanh', 'trace', 'transpose', 'true_divide',+    'var', 'where', 'zeros',+    ]++MaskType = np.bool_+nomask = MaskType(0)++class MaskedArrayFutureWarning(FutureWarning):+    pass++def _deprecate_argsort_axis(arr):+    """+    Adjust the axis passed to argsort, warning if necessary++    Parameters+    ----------+    arr+        The array which argsort was called on++    np.ma.argsort has a long-term bug where the default of the axis argument+    is wrong (gh-8701), which now must be kept for backwards compatibiity.+    Thankfully, this only makes a difference when arrays are 2- or more-+    dimensional, so we only need a warning then.+    """+    if arr.ndim <= 1:+        # no warning needed - but switch to -1 anyway, to avoid surprising+        # subclasses, which are more likely to implement scalar axes.+        return -1+    else:+        # 2017-04-11, Numpy 1.13.0, gh-8701: warn on axis default+        warnings.warn(+            "In the future the default for argsort will be axis=-1, not the "+            "current None, to match its documentation and np.argsort. "+            "Explicitly pass -1 or None to silence this warning.",+            MaskedArrayFutureWarning, stacklevel=3)+        return None+++def doc_note(initialdoc, note):+    """+    Adds a Notes section to an existing docstring.++    """+    if initialdoc is None:+        return+    if note is None:+        return initialdoc++    notesplit = re.split(r'\n\s*?Notes\n\s*?-----', initialdoc)++    notedoc = """\+Notes+    -----+    %s""" % note++    if len(notesplit) > 1:+        notedoc = '\n\n    ' + notedoc + '\n'++    return ''.join(notesplit[:1] + [notedoc] + notesplit[1:])+++def get_object_signature(obj):+    """+    Get the signature from obj++    """+    try:+        sig = formatargspec(*getargspec(obj))+    except TypeError:+        sig = ''+    return sig+++###############################################################################+#                              Exceptions                                     #+###############################################################################+++class MAError(Exception):+    """+    Class for masked array related errors.++    """+    pass+++class MaskError(MAError):+    """+    Class for mask related errors.++    """+    pass+++###############################################################################+#                           Filling options                                   #+###############################################################################+++# b: boolean - c: complex - f: floats - i: integer - O: object - S: string+default_filler = {'b': True,+                  'c': 1.e20 + 0.0j,+                  'f': 1.e20,+                  'i': 999999,+                  'O': '?',+                  'S': b'N/A',+                  'u': 999999,+                  'V': b'???',+                  'U': u'N/A'+                  }++# Add datetime64 and timedelta64 types+for v in ["Y", "M", "W", "D", "h", "m", "s", "ms", "us", "ns", "ps",+          "fs", "as"]:+    default_filler["M8[" + v + "]"] = np.datetime64("NaT", v)+    default_filler["m8[" + v + "]"] = np.timedelta64("NaT", v)++max_filler = ntypes._minvals+max_filler.update([(k, -np.inf) for k in [np.float32, np.float64]])+min_filler = ntypes._maxvals+min_filler.update([(k, +np.inf) for k in [np.float32, np.float64]])+if 'float128' in ntypes.typeDict:+    max_filler.update([(np.float128, -np.inf)])+    min_filler.update([(np.float128, +np.inf)])+++def _recursive_fill_value(dtype, f):+    """+    Recursively produce a fill value for `dtype`, calling f on scalar dtypes+    """+    if dtype.names is not None:+        vals = tuple(_recursive_fill_value(dtype[name], f) for name in dtype.names)+        return np.array(vals, dtype=dtype)[()]  # decay to void scalar from 0d+    elif dtype.subdtype:+        subtype, shape = dtype.subdtype+        subval = _recursive_fill_value(subtype, f)+        return np.full(shape, subval)+    else:+        return f(dtype)+++def _get_dtype_of(obj):+    """ Convert the argument for *_fill_value into a dtype """+    if isinstance(obj, np.dtype):+        return obj+    elif hasattr(obj, 'dtype'):+        return obj.dtype+    else:+        return np.asanyarray(obj).dtype+++def default_fill_value(obj):+    """+    Return the default fill value for the argument object.++    The default filling value depends on the datatype of the input+    array or the type of the input scalar:++       ========  ========+       datatype  default+       ========  ========+       bool      True+       int       999999+       float     1.e20+       complex   1.e20+0j+       object    '?'+       string    'N/A'+       ========  ========++    For structured types, a structured scalar is returned, with each field the+    default fill value for its type.++    For subarray types, the fill value is an array of the same size containing+    the default scalar fill value.++    Parameters+    ----------+    obj : ndarray, dtype or scalar+        The array data-type or scalar for which the default fill value+        is returned.++    Returns+    -------+    fill_value : scalar+        The default fill value.++    Examples+    --------+    >>> np.ma.default_fill_value(1)+    999999+    >>> np.ma.default_fill_value(np.array([1.1, 2., np.pi]))+    1e+20+    >>> np.ma.default_fill_value(np.dtype(complex))+    (1e+20+0j)++    """+    def _scalar_fill_value(dtype):+        if dtype.kind in 'Mm':+            return default_filler.get(dtype.str[1:], '?')+        else:+            return default_filler.get(dtype.kind, '?')++    dtype = _get_dtype_of(obj)+    return _recursive_fill_value(dtype, _scalar_fill_value)+++def _extremum_fill_value(obj, extremum, extremum_name):++    def _scalar_fill_value(dtype):+        try:+            return extremum[dtype]+        except KeyError:+            raise TypeError(+                "Unsuitable type {} for calculating {}."+                .format(dtype, extremum_name)+            )++    dtype = _get_dtype_of(obj)+    return _recursive_fill_value(dtype, _scalar_fill_value)+++def minimum_fill_value(obj):+    """+    Return the maximum value that can be represented by the dtype of an object.++    This function is useful for calculating a fill value suitable for+    taking the minimum of an array with a given dtype.++    Parameters+    ----------+    obj : ndarray, dtype or scalar+        An object that can be queried for it's numeric type.++    Returns+    -------+    val : scalar+        The maximum representable value.++    Raises+    ------+    TypeError+        If `obj` isn't a suitable numeric type.++    See Also+    --------+    maximum_fill_value : The inverse function.+    set_fill_value : Set the filling value of a masked array.+    MaskedArray.fill_value : Return current fill value.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.int8()+    >>> ma.minimum_fill_value(a)+    127+    >>> a = np.int32()+    >>> ma.minimum_fill_value(a)+    2147483647++    An array of numeric data can also be passed.++    >>> a = np.array([1, 2, 3], dtype=np.int8)+    >>> ma.minimum_fill_value(a)+    127+    >>> a = np.array([1, 2, 3], dtype=np.float32)+    >>> ma.minimum_fill_value(a)+    inf++    """+    return _extremum_fill_value(obj, min_filler, "minimum")+++def maximum_fill_value(obj):+    """+    Return the minimum value that can be represented by the dtype of an object.++    This function is useful for calculating a fill value suitable for+    taking the maximum of an array with a given dtype.++    Parameters+    ----------+    obj : ndarray, dtype or scalar+        An object that can be queried for it's numeric type.++    Returns+    -------+    val : scalar+        The minimum representable value.++    Raises+    ------+    TypeError+        If `obj` isn't a suitable numeric type.++    See Also+    --------+    minimum_fill_value : The inverse function.+    set_fill_value : Set the filling value of a masked array.+    MaskedArray.fill_value : Return current fill value.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.int8()+    >>> ma.maximum_fill_value(a)+    -128+    >>> a = np.int32()+    >>> ma.maximum_fill_value(a)+    -2147483648++    An array of numeric data can also be passed.++    >>> a = np.array([1, 2, 3], dtype=np.int8)+    >>> ma.maximum_fill_value(a)+    -128+    >>> a = np.array([1, 2, 3], dtype=np.float32)+    >>> ma.maximum_fill_value(a)+    -inf++    """+    return _extremum_fill_value(obj, max_filler, "maximum")+++def _recursive_set_fill_value(fillvalue, dt):+    """+    Create a fill value for a structured dtype.++    Parameters+    ----------+    fillvalue: scalar or array_like+        Scalar or array representing the fill value. If it is of shorter+        length than the number of fields in dt, it will be resized.+    dt: dtype+        The structured dtype for which to create the fill value.++    Returns+    -------+    val: tuple+        A tuple of values corresponding to the structured fill value.++    """+    fillvalue = np.resize(fillvalue, len(dt.names))+    output_value = []+    for (fval, name) in zip(fillvalue, dt.names):+        cdtype = dt[name]+        if cdtype.subdtype:+            cdtype = cdtype.subdtype[0]++        if cdtype.names is not None:+            output_value.append(tuple(_recursive_set_fill_value(fval, cdtype)))+        else:+            output_value.append(np.array(fval, dtype=cdtype).item())+    return tuple(output_value)+++def _check_fill_value(fill_value, ndtype):+    """+    Private function validating the given `fill_value` for the given dtype.++    If fill_value is None, it is set to the default corresponding to the dtype.++    If fill_value is not None, its value is forced to the given dtype.++    The result is always a 0d array.+    """+    ndtype = np.dtype(ndtype)+    fields = ndtype.fields+    if fill_value is None:+        fill_value = default_fill_value(ndtype)+    elif fields:+        fdtype = [(_[0], _[1]) for _ in ndtype.descr]+        if isinstance(fill_value, (ndarray, np.void)):+            try:+                fill_value = np.array(fill_value, copy=False, dtype=fdtype)+            except ValueError:+                err_msg = "Unable to transform %s to dtype %s"+                raise ValueError(err_msg % (fill_value, fdtype))+        else:+            fill_value = np.asarray(fill_value, dtype=object)+            fill_value = np.array(_recursive_set_fill_value(fill_value, ndtype),+                                  dtype=ndtype)+    else:+        if isinstance(fill_value, basestring) and (ndtype.char not in 'OSVU'):+            err_msg = "Cannot set fill value of string with array of dtype %s"+            raise TypeError(err_msg % ndtype)+        else:+            # In case we want to convert 1e20 to int.+            try:+                fill_value = np.array(fill_value, copy=False, dtype=ndtype)+            except OverflowError:+                # Raise TypeError instead of OverflowError. OverflowError+                # is seldom used, and the real problem here is that the+                # passed fill_value is not compatible with the ndtype.+                err_msg = "Fill value %s overflows dtype %s"+                raise TypeError(err_msg % (fill_value, ndtype))+    return np.array(fill_value)+++def set_fill_value(a, fill_value):+    """+    Set the filling value of a, if a is a masked array.++    This function changes the fill value of the masked array `a` in place.+    If `a` is not a masked array, the function returns silently, without+    doing anything.++    Parameters+    ----------+    a : array_like+        Input array.+    fill_value : dtype+        Filling value. A consistency test is performed to make sure+        the value is compatible with the dtype of `a`.++    Returns+    -------+    None+        Nothing returned by this function.++    See Also+    --------+    maximum_fill_value : Return the default fill value for a dtype.+    MaskedArray.fill_value : Return current fill value.+    MaskedArray.set_fill_value : Equivalent method.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(5)+    >>> a+    array([0, 1, 2, 3, 4])+    >>> a = ma.masked_where(a < 3, a)+    >>> a+    masked_array(data = [-- -- -- 3 4],+          mask = [ True  True  True False False],+          fill_value=999999)+    >>> ma.set_fill_value(a, -999)+    >>> a+    masked_array(data = [-- -- -- 3 4],+          mask = [ True  True  True False False],+          fill_value=-999)++    Nothing happens if `a` is not a masked array.++    >>> a = range(5)+    >>> a+    [0, 1, 2, 3, 4]+    >>> ma.set_fill_value(a, 100)+    >>> a+    [0, 1, 2, 3, 4]+    >>> a = np.arange(5)+    >>> a+    array([0, 1, 2, 3, 4])+    >>> ma.set_fill_value(a, 100)+    >>> a+    array([0, 1, 2, 3, 4])++    """+    if isinstance(a, MaskedArray):+        a.set_fill_value(fill_value)+    return+++def get_fill_value(a):+    """+    Return the filling value of a, if any.  Otherwise, returns the+    default filling value for that type.++    """+    if isinstance(a, MaskedArray):+        result = a.fill_value+    else:+        result = default_fill_value(a)+    return result+++def common_fill_value(a, b):+    """+    Return the common filling value of two masked arrays, if any.++    If ``a.fill_value == b.fill_value``, return the fill value,+    otherwise return None.++    Parameters+    ----------+    a, b : MaskedArray+        The masked arrays for which to compare fill values.++    Returns+    -------+    fill_value : scalar or None+        The common fill value, or None.++    Examples+    --------+    >>> x = np.ma.array([0, 1.], fill_value=3)+    >>> y = np.ma.array([0, 1.], fill_value=3)+    >>> np.ma.common_fill_value(x, y)+    3.0++    """+    t1 = get_fill_value(a)+    t2 = get_fill_value(b)+    if t1 == t2:+        return t1+    return None+++def filled(a, fill_value=None):+    """+    Return input as an array with masked data replaced by a fill value.++    If `a` is not a `MaskedArray`, `a` itself is returned.+    If `a` is a `MaskedArray` and `fill_value` is None, `fill_value` is set to+    ``a.fill_value``.++    Parameters+    ----------+    a : MaskedArray or array_like+        An input object.+    fill_value : scalar, optional+        Filling value. Default is None.++    Returns+    -------+    a : ndarray+        The filled array.++    See Also+    --------+    compressed++    Examples+    --------+    >>> x = np.ma.array(np.arange(9).reshape(3, 3), mask=[[1, 0, 0],+    ...                                                   [1, 0, 0],+    ...                                                   [0, 0, 0]])+    >>> x.filled()+    array([[999999,      1,      2],+           [999999,      4,      5],+           [     6,      7,      8]])++    """+    if hasattr(a, 'filled'):+        return a.filled(fill_value)+    elif isinstance(a, ndarray):+        # Should we check for contiguity ? and a.flags['CONTIGUOUS']:+        return a+    elif isinstance(a, dict):+        return np.array(a, 'O')+    else:+        return np.array(a)+++def get_masked_subclass(*arrays):+    """+    Return the youngest subclass of MaskedArray from a list of (masked) arrays.++    In case of siblings, the first listed takes over.++    """+    if len(arrays) == 1:+        arr = arrays[0]+        if isinstance(arr, MaskedArray):+            rcls = type(arr)+        else:+            rcls = MaskedArray+    else:+        arrcls = [type(a) for a in arrays]+        rcls = arrcls[0]+        if not issubclass(rcls, MaskedArray):+            rcls = MaskedArray+        for cls in arrcls[1:]:+            if issubclass(cls, rcls):+                rcls = cls+    # Don't return MaskedConstant as result: revert to MaskedArray+    if rcls.__name__ == 'MaskedConstant':+        return MaskedArray+    return rcls+++def getdata(a, subok=True):+    """+    Return the data of a masked array as an ndarray.++    Return the data of `a` (if any) as an ndarray if `a` is a ``MaskedArray``,+    else return `a` as a ndarray or subclass (depending on `subok`) if not.++    Parameters+    ----------+    a : array_like+        Input ``MaskedArray``, alternatively a ndarray or a subclass thereof.+    subok : bool+        Whether to force the output to be a `pure` ndarray (False) or to+        return a subclass of ndarray if appropriate (True, default).++    See Also+    --------+    getmask : Return the mask of a masked array, or nomask.+    getmaskarray : Return the mask of a masked array, or full array of False.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = ma.masked_equal([[1,2],[3,4]], 2)+    >>> a+    masked_array(data =+     [[1 --]+     [3 4]],+          mask =+     [[False  True]+     [False False]],+          fill_value=999999)+    >>> ma.getdata(a)+    array([[1, 2],+           [3, 4]])++    Equivalently use the ``MaskedArray`` `data` attribute.++    >>> a.data+    array([[1, 2],+           [3, 4]])++    """+    try:+        data = a._data+    except AttributeError:+        data = np.array(a, copy=False, subok=subok)+    if not subok:+        return data.view(ndarray)+    return data+++get_data = getdata+++def fix_invalid(a, mask=nomask, copy=True, fill_value=None):+    """+    Return input with invalid data masked and replaced by a fill value.++    Invalid data means values of `nan`, `inf`, etc.++    Parameters+    ----------+    a : array_like+        Input array, a (subclass of) ndarray.+    mask : sequence, optional+        Mask. Must be convertible to an array of booleans with the same+        shape as `data`. True indicates a masked (i.e. invalid) data.+    copy : bool, optional+        Whether to use a copy of `a` (True) or to fix `a` in place (False).+        Default is True.+    fill_value : scalar, optional+        Value used for fixing invalid data. Default is None, in which case+        the ``a.fill_value`` is used.++    Returns+    -------+    b : MaskedArray+        The input array with invalid entries fixed.++    Notes+    -----+    A copy is performed by default.++    Examples+    --------+    >>> x = np.ma.array([1., -1, np.nan, np.inf], mask=[1] + [0]*3)+    >>> x+    masked_array(data = [-- -1.0 nan inf],+                 mask = [ True False False False],+           fill_value = 1e+20)+    >>> np.ma.fix_invalid(x)+    masked_array(data = [-- -1.0 -- --],+                 mask = [ True False  True  True],+           fill_value = 1e+20)++    >>> fixed = np.ma.fix_invalid(x)+    >>> fixed.data+    array([  1.00000000e+00,  -1.00000000e+00,   1.00000000e+20,+             1.00000000e+20])+    >>> x.data+    array([  1.,  -1.,  NaN,  Inf])++    """+    a = masked_array(a, copy=copy, mask=mask, subok=True)+    invalid = np.logical_not(np.isfinite(a._data))+    if not invalid.any():+        return a+    a._mask |= invalid+    if fill_value is None:+        fill_value = a.fill_value+    a._data[invalid] = fill_value+    return a+++###############################################################################+#                                  Ufuncs                                     #+###############################################################################+++ufunc_domain = {}+ufunc_fills = {}+++class _DomainCheckInterval(object):+    """+    Define a valid interval, so that :++    ``domain_check_interval(a,b)(x) == True`` where+    ``x < a`` or ``x > b``.++    """++    def __init__(self, a, b):+        "domain_check_interval(a,b)(x) = true where x < a or y > b"+        if (a > b):+            (a, b) = (b, a)+        self.a = a+        self.b = b++    def __call__(self, x):+        "Execute the call behavior."+        # nans at masked positions cause RuntimeWarnings, even though+        # they are masked. To avoid this we suppress warnings.+        with np.errstate(invalid='ignore'):+            return umath.logical_or(umath.greater(x, self.b),+                                    umath.less(x, self.a))+++class _DomainTan(object):+    """+    Define a valid interval for the `tan` function, so that:++    ``domain_tan(eps) = True`` where ``abs(cos(x)) < eps``++    """++    def __init__(self, eps):+        "domain_tan(eps) = true where abs(cos(x)) < eps)"+        self.eps = eps++    def __call__(self, x):+        "Executes the call behavior."+        with np.errstate(invalid='ignore'):+            return umath.less(umath.absolute(umath.cos(x)), self.eps)+++class _DomainSafeDivide(object):+    """+    Define a domain for safe division.++    """++    def __init__(self, tolerance=None):+        self.tolerance = tolerance++    def __call__(self, a, b):+        # Delay the selection of the tolerance to here in order to reduce numpy+        # import times. The calculation of these parameters is a substantial+        # component of numpy's import time.+        if self.tolerance is None:+            self.tolerance = np.finfo(float).tiny+        # don't call ma ufuncs from __array_wrap__ which would fail for scalars+        a, b = np.asarray(a), np.asarray(b)+        with np.errstate(invalid='ignore'):+            return umath.absolute(a) * self.tolerance >= umath.absolute(b)+++class _DomainGreater(object):+    """+    DomainGreater(v)(x) is True where x <= v.++    """++    def __init__(self, critical_value):+        "DomainGreater(v)(x) = true where x <= v"+        self.critical_value = critical_value++    def __call__(self, x):+        "Executes the call behavior."+        with np.errstate(invalid='ignore'):+            return umath.less_equal(x, self.critical_value)+++class _DomainGreaterEqual(object):+    """+    DomainGreaterEqual(v)(x) is True where x < v.++    """++    def __init__(self, critical_value):+        "DomainGreaterEqual(v)(x) = true where x < v"+        self.critical_value = critical_value++    def __call__(self, x):+        "Executes the call behavior."+        with np.errstate(invalid='ignore'):+            return umath.less(x, self.critical_value)+++class _MaskedUFunc(object):+    def __init__(self, ufunc):+        self.f = ufunc+        self.__doc__ = ufunc.__doc__+        self.__name__ = ufunc.__name__++    def __str__(self):+        return "Masked version of {}".format(self.f)+++class _MaskedUnaryOperation(_MaskedUFunc):+    """+    Defines masked version of unary operations, where invalid values are+    pre-masked.++    Parameters+    ----------+    mufunc : callable+        The function for which to define a masked version. Made available+        as ``_MaskedUnaryOperation.f``.+    fill : scalar, optional+        Filling value, default is 0.+    domain : class instance+        Domain for the function. Should be one of the ``_Domain*``+        classes. Default is None.++    """++    def __init__(self, mufunc, fill=0, domain=None):+        super(_MaskedUnaryOperation, self).__init__(mufunc)+        self.fill = fill+        self.domain = domain+        ufunc_domain[mufunc] = domain+        ufunc_fills[mufunc] = fill++    def __call__(self, a, *args, **kwargs):+        """+        Execute the call behavior.++        """+        d = getdata(a)+        # Deal with domain+        if self.domain is not None:+            # Case 1.1. : Domained function+            # nans at masked positions cause RuntimeWarnings, even though+            # they are masked. To avoid this we suppress warnings.+            with np.errstate(divide='ignore', invalid='ignore'):+                result = self.f(d, *args, **kwargs)+            # Make a mask+            m = ~umath.isfinite(result)+            m |= self.domain(d)+            m |= getmask(a)+        else:+            # Case 1.2. : Function without a domain+            # Get the result and the mask+            with np.errstate(divide='ignore', invalid='ignore'):+                result = self.f(d, *args, **kwargs)+            m = getmask(a)++        if not result.ndim:+            # Case 2.1. : The result is scalarscalar+            if m:+                return masked+            return result++        if m is not nomask:+            # Case 2.2. The result is an array+            # We need to fill the invalid data back w/ the input Now,+            # that's plain silly: in C, we would just skip the element and+            # keep the original, but we do have to do it that way in Python++            # In case result has a lower dtype than the inputs (as in+            # equal)+            try:+                np.copyto(result, d, where=m)+            except TypeError:+                pass+        # Transform to+        masked_result = result.view(get_masked_subclass(a))+        masked_result._mask = m+        masked_result._update_from(a)+        return masked_result+++class _MaskedBinaryOperation(_MaskedUFunc):+    """+    Define masked version of binary operations, where invalid+    values are pre-masked.++    Parameters+    ----------+    mbfunc : function+        The function for which to define a masked version. Made available+        as ``_MaskedBinaryOperation.f``.+    domain : class instance+        Default domain for the function. Should be one of the ``_Domain*``+        classes. Default is None.+    fillx : scalar, optional+        Filling value for the first argument, default is 0.+    filly : scalar, optional+        Filling value for the second argument, default is 0.++    """++    def __init__(self, mbfunc, fillx=0, filly=0):+        """+        abfunc(fillx, filly) must be defined.++        abfunc(x, filly) = x for all x to enable reduce.++        """+        super(_MaskedBinaryOperation, self).__init__(mbfunc)+        self.fillx = fillx+        self.filly = filly+        ufunc_domain[mbfunc] = None+        ufunc_fills[mbfunc] = (fillx, filly)++    def __call__(self, a, b, *args, **kwargs):+        """+        Execute the call behavior.++        """+        # Get the data, as ndarray+        (da, db) = (getdata(a), getdata(b))+        # Get the result+        with np.errstate():+            np.seterr(divide='ignore', invalid='ignore')+            result = self.f(da, db, *args, **kwargs)+        # Get the mask for the result+        (ma, mb) = (getmask(a), getmask(b))+        if ma is nomask:+            if mb is nomask:+                m = nomask+            else:+                m = umath.logical_or(getmaskarray(a), mb)+        elif mb is nomask:+            m = umath.logical_or(ma, getmaskarray(b))+        else:+            m = umath.logical_or(ma, mb)++        # Case 1. : scalar+        if not result.ndim:+            if m:+                return masked+            return result++        # Case 2. : array+        # Revert result to da where masked+        if m is not nomask and m.any():+            # any errors, just abort; impossible to guarantee masked values+            try:+                np.copyto(result, da, casting='unsafe', where=m)+            except Exception:+                pass++        # Transforms to a (subclass of) MaskedArray+        masked_result = result.view(get_masked_subclass(a, b))+        masked_result._mask = m+        if isinstance(a, MaskedArray):+            masked_result._update_from(a)+        elif isinstance(b, MaskedArray):+            masked_result._update_from(b)+        return masked_result++    def reduce(self, target, axis=0, dtype=None):+        """+        Reduce `target` along the given `axis`.++        """+        tclass = get_masked_subclass(target)+        m = getmask(target)+        t = filled(target, self.filly)+        if t.shape == ():+            t = t.reshape(1)+            if m is not nomask:+                m = make_mask(m, copy=1)+                m.shape = (1,)++        if m is nomask:+            tr = self.f.reduce(t, axis)+            mr = nomask+        else:+            tr = self.f.reduce(t, axis, dtype=dtype or t.dtype)+            mr = umath.logical_and.reduce(m, axis)++        if not tr.shape:+            if mr:+                return masked+            else:+                return tr+        masked_tr = tr.view(tclass)+        masked_tr._mask = mr+        return masked_tr++    def outer(self, a, b):+        """+        Return the function applied to the outer product of a and b.++        """+        (da, db) = (getdata(a), getdata(b))+        d = self.f.outer(da, db)+        ma = getmask(a)+        mb = getmask(b)+        if ma is nomask and mb is nomask:+            m = nomask+        else:+            ma = getmaskarray(a)+            mb = getmaskarray(b)+            m = umath.logical_or.outer(ma, mb)+        if (not m.ndim) and m:+            return masked+        if m is not nomask:+            np.copyto(d, da, where=m)+        if not d.shape:+            return d+        masked_d = d.view(get_masked_subclass(a, b))+        masked_d._mask = m+        return masked_d++    def accumulate(self, target, axis=0):+        """Accumulate `target` along `axis` after filling with y fill+        value.++        """+        tclass = get_masked_subclass(target)+        t = filled(target, self.filly)+        result = self.f.accumulate(t, axis)+        masked_result = result.view(tclass)+        return masked_result++++class _DomainedBinaryOperation(_MaskedUFunc):+    """+    Define binary operations that have a domain, like divide.++    They have no reduce, outer or accumulate.++    Parameters+    ----------+    mbfunc : function+        The function for which to define a masked version. Made available+        as ``_DomainedBinaryOperation.f``.+    domain : class instance+        Default domain for the function. Should be one of the ``_Domain*``+        classes.+    fillx : scalar, optional+        Filling value for the first argument, default is 0.+    filly : scalar, optional+        Filling value for the second argument, default is 0.++    """++    def __init__(self, dbfunc, domain, fillx=0, filly=0):+        """abfunc(fillx, filly) must be defined.+           abfunc(x, filly) = x for all x to enable reduce.+        """+        super(_DomainedBinaryOperation, self).__init__(dbfunc)+        self.domain = domain+        self.fillx = fillx+        self.filly = filly+        ufunc_domain[dbfunc] = domain+        ufunc_fills[dbfunc] = (fillx, filly)++    def __call__(self, a, b, *args, **kwargs):+        "Execute the call behavior."+        # Get the data+        (da, db) = (getdata(a), getdata(b))+        # Get the result+        with np.errstate(divide='ignore', invalid='ignore'):+            result = self.f(da, db, *args, **kwargs)+        # Get the mask as a combination of the source masks and invalid+        m = ~umath.isfinite(result)+        m |= getmask(a)+        m |= getmask(b)+        # Apply the domain+        domain = ufunc_domain.get(self.f, None)+        if domain is not None:+            m |= domain(da, db)+        # Take care of the scalar case first+        if (not m.ndim):+            if m:+                return masked+            else:+                return result+        # When the mask is True, put back da if possible+        # any errors, just abort; impossible to guarantee masked values+        try:+            np.copyto(result, 0, casting='unsafe', where=m)+            # avoid using "*" since this may be overlaid+            masked_da = umath.multiply(m, da)+            # only add back if it can be cast safely+            if np.can_cast(masked_da.dtype, result.dtype, casting='safe'):+                result += masked_da+        except Exception:+            pass++        # Transforms to a (subclass of) MaskedArray+        masked_result = result.view(get_masked_subclass(a, b))+        masked_result._mask = m+        if isinstance(a, MaskedArray):+            masked_result._update_from(a)+        elif isinstance(b, MaskedArray):+            masked_result._update_from(b)+        return masked_result+++# Unary ufuncs+exp = _MaskedUnaryOperation(umath.exp)+conjugate = _MaskedUnaryOperation(umath.conjugate)+sin = _MaskedUnaryOperation(umath.sin)+cos = _MaskedUnaryOperation(umath.cos)+tan = _MaskedUnaryOperation(umath.tan)+arctan = _MaskedUnaryOperation(umath.arctan)+arcsinh = _MaskedUnaryOperation(umath.arcsinh)+sinh = _MaskedUnaryOperation(umath.sinh)+cosh = _MaskedUnaryOperation(umath.cosh)+tanh = _MaskedUnaryOperation(umath.tanh)+abs = absolute = _MaskedUnaryOperation(umath.absolute)+angle = _MaskedUnaryOperation(angle)  # from numpy.lib.function_base+fabs = _MaskedUnaryOperation(umath.fabs)+negative = _MaskedUnaryOperation(umath.negative)+floor = _MaskedUnaryOperation(umath.floor)+ceil = _MaskedUnaryOperation(umath.ceil)+around = _MaskedUnaryOperation(np.round_)+logical_not = _MaskedUnaryOperation(umath.logical_not)++# Domained unary ufuncs+sqrt = _MaskedUnaryOperation(umath.sqrt, 0.0,+                             _DomainGreaterEqual(0.0))+log = _MaskedUnaryOperation(umath.log, 1.0,+                            _DomainGreater(0.0))+log2 = _MaskedUnaryOperation(umath.log2, 1.0,+                             _DomainGreater(0.0))+log10 = _MaskedUnaryOperation(umath.log10, 1.0,+                              _DomainGreater(0.0))+tan = _MaskedUnaryOperation(umath.tan, 0.0,+                            _DomainTan(1e-35))+arcsin = _MaskedUnaryOperation(umath.arcsin, 0.0,+                               _DomainCheckInterval(-1.0, 1.0))+arccos = _MaskedUnaryOperation(umath.arccos, 0.0,+                               _DomainCheckInterval(-1.0, 1.0))+arccosh = _MaskedUnaryOperation(umath.arccosh, 1.0,+                                _DomainGreaterEqual(1.0))+arctanh = _MaskedUnaryOperation(umath.arctanh, 0.0,+                                _DomainCheckInterval(-1.0 + 1e-15, 1.0 - 1e-15))++# Binary ufuncs+add = _MaskedBinaryOperation(umath.add)+subtract = _MaskedBinaryOperation(umath.subtract)+multiply = _MaskedBinaryOperation(umath.multiply, 1, 1)+arctan2 = _MaskedBinaryOperation(umath.arctan2, 0.0, 1.0)+equal = _MaskedBinaryOperation(umath.equal)+equal.reduce = None+not_equal = _MaskedBinaryOperation(umath.not_equal)+not_equal.reduce = None+less_equal = _MaskedBinaryOperation(umath.less_equal)+less_equal.reduce = None+greater_equal = _MaskedBinaryOperation(umath.greater_equal)+greater_equal.reduce = None+less = _MaskedBinaryOperation(umath.less)+less.reduce = None+greater = _MaskedBinaryOperation(umath.greater)+greater.reduce = None+logical_and = _MaskedBinaryOperation(umath.logical_and)+alltrue = _MaskedBinaryOperation(umath.logical_and, 1, 1).reduce+logical_or = _MaskedBinaryOperation(umath.logical_or)+sometrue = logical_or.reduce+logical_xor = _MaskedBinaryOperation(umath.logical_xor)+bitwise_and = _MaskedBinaryOperation(umath.bitwise_and)+bitwise_or = _MaskedBinaryOperation(umath.bitwise_or)+bitwise_xor = _MaskedBinaryOperation(umath.bitwise_xor)+hypot = _MaskedBinaryOperation(umath.hypot)++# Domained binary ufuncs+divide = _DomainedBinaryOperation(umath.divide, _DomainSafeDivide(), 0, 1)+true_divide = _DomainedBinaryOperation(umath.true_divide,+                                       _DomainSafeDivide(), 0, 1)+floor_divide = _DomainedBinaryOperation(umath.floor_divide,+                                        _DomainSafeDivide(), 0, 1)+remainder = _DomainedBinaryOperation(umath.remainder,+                                     _DomainSafeDivide(), 0, 1)+fmod = _DomainedBinaryOperation(umath.fmod, _DomainSafeDivide(), 0, 1)+mod = _DomainedBinaryOperation(umath.mod, _DomainSafeDivide(), 0, 1)+++###############################################################################+#                        Mask creation functions                              #+###############################################################################+++def _replace_dtype_fields_recursive(dtype, primitive_dtype):+    "Private function allowing recursion in _replace_dtype_fields."+    _recurse = _replace_dtype_fields_recursive++    # Do we have some name fields ?+    if dtype.names is not None:+        descr = []+        for name in dtype.names:+            field = dtype.fields[name]+            if len(field) == 3:+                # Prepend the title to the name+                name = (field[-1], name)+            descr.append((name, _recurse(field[0], primitive_dtype)))+        new_dtype = np.dtype(descr)++    # Is this some kind of composite a la (float,2)+    elif dtype.subdtype:+        descr = list(dtype.subdtype)+        descr[0] = _recurse(dtype.subdtype[0], primitive_dtype)+        new_dtype = np.dtype(tuple(descr))++    # this is a primitive type, so do a direct replacement+    else:+        new_dtype = primitive_dtype++    # preserve identity of dtypes+    if new_dtype == dtype:+        new_dtype = dtype++    return new_dtype+++def _replace_dtype_fields(dtype, primitive_dtype):+    """+    Construct a dtype description list from a given dtype.++    Returns a new dtype object, with all fields and subtypes in the given type+    recursively replaced with `primitive_dtype`.++    Arguments are coerced to dtypes first.+    """+    dtype = np.dtype(dtype)+    primitive_dtype = np.dtype(primitive_dtype)+    return _replace_dtype_fields_recursive(dtype, primitive_dtype)+++def make_mask_descr(ndtype):+    """+    Construct a dtype description list from a given dtype.++    Returns a new dtype object, with the type of all fields in `ndtype` to a+    boolean type. Field names are not altered.++    Parameters+    ----------+    ndtype : dtype+        The dtype to convert.++    Returns+    -------+    result : dtype+        A dtype that looks like `ndtype`, the type of all fields is boolean.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> dtype = np.dtype({'names':['foo', 'bar'],+                          'formats':[np.float32, int]})+    >>> dtype+    dtype([('foo', '<f4'), ('bar', '<i4')])+    >>> ma.make_mask_descr(dtype)+    dtype([('foo', '|b1'), ('bar', '|b1')])+    >>> ma.make_mask_descr(np.float32)+    dtype('bool')++    """+    return _replace_dtype_fields(ndtype, MaskType)+++def getmask(a):+    """+    Return the mask of a masked array, or nomask.++    Return the mask of `a` as an ndarray if `a` is a `MaskedArray` and the+    mask is not `nomask`, else return `nomask`. To guarantee a full array+    of booleans of the same shape as a, use `getmaskarray`.++    Parameters+    ----------+    a : array_like+        Input `MaskedArray` for which the mask is required.++    See Also+    --------+    getdata : Return the data of a masked array as an ndarray.+    getmaskarray : Return the mask of a masked array, or full array of False.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = ma.masked_equal([[1,2],[3,4]], 2)+    >>> a+    masked_array(data =+     [[1 --]+     [3 4]],+          mask =+     [[False  True]+     [False False]],+          fill_value=999999)+    >>> ma.getmask(a)+    array([[False,  True],+           [False, False]])++    Equivalently use the `MaskedArray` `mask` attribute.++    >>> a.mask+    array([[False,  True],+           [False, False]])++    Result when mask == `nomask`++    >>> b = ma.masked_array([[1,2],[3,4]])+    >>> b+    masked_array(data =+     [[1 2]+     [3 4]],+          mask =+     False,+          fill_value=999999)+    >>> ma.nomask+    False+    >>> ma.getmask(b) == ma.nomask+    True+    >>> b.mask == ma.nomask+    True++    """+    return getattr(a, '_mask', nomask)+++get_mask = getmask+++def getmaskarray(arr):+    """+    Return the mask of a masked array, or full boolean array of False.++    Return the mask of `arr` as an ndarray if `arr` is a `MaskedArray` and+    the mask is not `nomask`, else return a full boolean array of False of+    the same shape as `arr`.++    Parameters+    ----------+    arr : array_like+        Input `MaskedArray` for which the mask is required.++    See Also+    --------+    getmask : Return the mask of a masked array, or nomask.+    getdata : Return the data of a masked array as an ndarray.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = ma.masked_equal([[1,2],[3,4]], 2)+    >>> a+    masked_array(data =+     [[1 --]+     [3 4]],+          mask =+     [[False  True]+     [False False]],+          fill_value=999999)+    >>> ma.getmaskarray(a)+    array([[False,  True],+           [False, False]])++    Result when mask == ``nomask``++    >>> b = ma.masked_array([[1,2],[3,4]])+    >>> b+    masked_array(data =+     [[1 2]+     [3 4]],+          mask =+     False,+          fill_value=999999)+    >>> >ma.getmaskarray(b)+    array([[False, False],+           [False, False]])++    """+    mask = getmask(arr)+    if mask is nomask:+        mask = make_mask_none(np.shape(arr), getattr(arr, 'dtype', None))+    return mask+++def is_mask(m):+    """+    Return True if m is a valid, standard mask.++    This function does not check the contents of the input, only that the+    type is MaskType. In particular, this function returns False if the+    mask has a flexible dtype.++    Parameters+    ----------+    m : array_like+        Array to test.++    Returns+    -------+    result : bool+        True if `m.dtype.type` is MaskType, False otherwise.++    See Also+    --------+    isMaskedArray : Test whether input is an instance of MaskedArray.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> m = ma.masked_equal([0, 1, 0, 2, 3], 0)+    >>> m+    masked_array(data = [-- 1 -- 2 3],+          mask = [ True False  True False False],+          fill_value=999999)+    >>> ma.is_mask(m)+    False+    >>> ma.is_mask(m.mask)+    True++    Input must be an ndarray (or have similar attributes)+    for it to be considered a valid mask.++    >>> m = [False, True, False]+    >>> ma.is_mask(m)+    False+    >>> m = np.array([False, True, False])+    >>> m+    array([False,  True, False])+    >>> ma.is_mask(m)+    True++    Arrays with complex dtypes don't return True.++    >>> dtype = np.dtype({'names':['monty', 'pithon'],+                          'formats':[bool, bool]})+    >>> dtype+    dtype([('monty', '|b1'), ('pithon', '|b1')])+    >>> m = np.array([(True, False), (False, True), (True, False)],+                     dtype=dtype)+    >>> m+    array([(True, False), (False, True), (True, False)],+          dtype=[('monty', '|b1'), ('pithon', '|b1')])+    >>> ma.is_mask(m)+    False++    """+    try:+        return m.dtype.type is MaskType+    except AttributeError:+        return False+++def _shrink_mask(m):+    """+    Shrink a mask to nomask if possible+    """+    if m.dtype.names is None and not m.any():+        return nomask+    else:+        return m+++def make_mask(m, copy=False, shrink=True, dtype=MaskType):+    """+    Create a boolean mask from an array.++    Return `m` as a boolean mask, creating a copy if necessary or requested.+    The function can accept any sequence that is convertible to integers,+    or ``nomask``.  Does not require that contents must be 0s and 1s, values+    of 0 are interepreted as False, everything else as True.++    Parameters+    ----------+    m : array_like+        Potential mask.+    copy : bool, optional+        Whether to return a copy of `m` (True) or `m` itself (False).+    shrink : bool, optional+        Whether to shrink `m` to ``nomask`` if all its values are False.+    dtype : dtype, optional+        Data-type of the output mask. By default, the output mask has a+        dtype of MaskType (bool). If the dtype is flexible, each field has+        a boolean dtype. This is ignored when `m` is ``nomask``, in which+        case ``nomask`` is always returned.++    Returns+    -------+    result : ndarray+        A boolean mask derived from `m`.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> m = [True, False, True, True]+    >>> ma.make_mask(m)+    array([ True, False,  True,  True])+    >>> m = [1, 0, 1, 1]+    >>> ma.make_mask(m)+    array([ True, False,  True,  True])+    >>> m = [1, 0, 2, -3]+    >>> ma.make_mask(m)+    array([ True, False,  True,  True])++    Effect of the `shrink` parameter.++    >>> m = np.zeros(4)+    >>> m+    array([ 0.,  0.,  0.,  0.])+    >>> ma.make_mask(m)+    False+    >>> ma.make_mask(m, shrink=False)+    array([False, False, False, False])++    Using a flexible `dtype`.++    >>> m = [1, 0, 1, 1]+    >>> n = [0, 1, 0, 0]+    >>> arr = []+    >>> for man, mouse in zip(m, n):+    ...     arr.append((man, mouse))+    >>> arr+    [(1, 0), (0, 1), (1, 0), (1, 0)]+    >>> dtype = np.dtype({'names':['man', 'mouse'],+                          'formats':[int, int]})+    >>> arr = np.array(arr, dtype=dtype)+    >>> arr+    array([(1, 0), (0, 1), (1, 0), (1, 0)],+          dtype=[('man', '<i4'), ('mouse', '<i4')])+    >>> ma.make_mask(arr, dtype=dtype)+    array([(True, False), (False, True), (True, False), (True, False)],+          dtype=[('man', '|b1'), ('mouse', '|b1')])++    """+    if m is nomask:+        return nomask++    # Make sure the input dtype is valid.+    dtype = make_mask_descr(dtype)++    # legacy boolean special case: "existence of fields implies true"+    if isinstance(m, ndarray) and m.dtype.fields and dtype == np.bool_:+        return np.ones(m.shape, dtype=dtype)++    # Fill the mask in case there are missing data; turn it into an ndarray.+    result = np.array(filled(m, True), copy=copy, dtype=dtype, subok=True)+    # Bas les masques !+    if shrink:+        result = _shrink_mask(result)+    return result+++def make_mask_none(newshape, dtype=None):+    """+    Return a boolean mask of the given shape, filled with False.++    This function returns a boolean ndarray with all entries False, that can+    be used in common mask manipulations. If a complex dtype is specified, the+    type of each field is converted to a boolean type.++    Parameters+    ----------+    newshape : tuple+        A tuple indicating the shape of the mask.+    dtype : {None, dtype}, optional+        If None, use a MaskType instance. Otherwise, use a new datatype with+        the same fields as `dtype`, converted to boolean types.++    Returns+    -------+    result : ndarray+        An ndarray of appropriate shape and dtype, filled with False.++    See Also+    --------+    make_mask : Create a boolean mask from an array.+    make_mask_descr : Construct a dtype description list from a given dtype.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> ma.make_mask_none((3,))+    array([False, False, False])++    Defining a more complex dtype.++    >>> dtype = np.dtype({'names':['foo', 'bar'],+                          'formats':[np.float32, int]})+    >>> dtype+    dtype([('foo', '<f4'), ('bar', '<i4')])+    >>> ma.make_mask_none((3,), dtype=dtype)+    array([(False, False), (False, False), (False, False)],+          dtype=[('foo', '|b1'), ('bar', '|b1')])++    """+    if dtype is None:+        result = np.zeros(newshape, dtype=MaskType)+    else:+        result = np.zeros(newshape, dtype=make_mask_descr(dtype))+    return result+++def mask_or(m1, m2, copy=False, shrink=True):+    """+    Combine two masks with the ``logical_or`` operator.++    The result may be a view on `m1` or `m2` if the other is `nomask`+    (i.e. False).++    Parameters+    ----------+    m1, m2 : array_like+        Input masks.+    copy : bool, optional+        If copy is False and one of the inputs is `nomask`, return a view+        of the other input mask. Defaults to False.+    shrink : bool, optional+        Whether to shrink the output to `nomask` if all its values are+        False. Defaults to True.++    Returns+    -------+    mask : output mask+        The result masks values that are masked in either `m1` or `m2`.++    Raises+    ------+    ValueError+        If `m1` and `m2` have different flexible dtypes.++    Examples+    --------+    >>> m1 = np.ma.make_mask([0, 1, 1, 0])+    >>> m2 = np.ma.make_mask([1, 0, 0, 0])+    >>> np.ma.mask_or(m1, m2)+    array([ True,  True,  True, False])++    """++    def _recursive_mask_or(m1, m2, newmask):+        names = m1.dtype.names+        for name in names:+            current1 = m1[name]+            if current1.dtype.names is not None:+                _recursive_mask_or(current1, m2[name], newmask[name])+            else:+                umath.logical_or(current1, m2[name], newmask[name])+        return++    if (m1 is nomask) or (m1 is False):+        dtype = getattr(m2, 'dtype', MaskType)+        return make_mask(m2, copy=copy, shrink=shrink, dtype=dtype)+    if (m2 is nomask) or (m2 is False):+        dtype = getattr(m1, 'dtype', MaskType)+        return make_mask(m1, copy=copy, shrink=shrink, dtype=dtype)+    if m1 is m2 and is_mask(m1):+        return m1+    (dtype1, dtype2) = (getattr(m1, 'dtype', None), getattr(m2, 'dtype', None))+    if (dtype1 != dtype2):+        raise ValueError("Incompatible dtypes '%s'<>'%s'" % (dtype1, dtype2))+    if dtype1.names is not None:+        # Allocate an output mask array with the properly broadcast shape.+        newmask = np.empty(np.broadcast(m1, m2).shape, dtype1)+        _recursive_mask_or(m1, m2, newmask)+        return newmask+    return make_mask(umath.logical_or(m1, m2), copy=copy, shrink=shrink)+++def flatten_mask(mask):+    """+    Returns a completely flattened version of the mask, where nested fields+    are collapsed.++    Parameters+    ----------+    mask : array_like+        Input array, which will be interpreted as booleans.++    Returns+    -------+    flattened_mask : ndarray of bools+        The flattened input.++    Examples+    --------+    >>> mask = np.array([0, 0, 1])+    >>> flatten_mask(mask)+    array([False, False,  True])++    >>> mask = np.array([(0, 0), (0, 1)], dtype=[('a', bool), ('b', bool)])+    >>> flatten_mask(mask)+    array([False, False, False,  True])++    >>> mdtype = [('a', bool), ('b', [('ba', bool), ('bb', bool)])]+    >>> mask = np.array([(0, (0, 0)), (0, (0, 1))], dtype=mdtype)+    >>> flatten_mask(mask)+    array([False, False, False, False, False,  True])++    """++    def _flatmask(mask):+        "Flatten the mask and returns a (maybe nested) sequence of booleans."+        mnames = mask.dtype.names+        if mnames is not None:+            return [flatten_mask(mask[name]) for name in mnames]+        else:+            return mask++    def _flatsequence(sequence):+        "Generates a flattened version of the sequence."+        try:+            for element in sequence:+                if hasattr(element, '__iter__'):+                    for f in _flatsequence(element):+                        yield f+                else:+                    yield element+        except TypeError:+            yield sequence++    mask = np.asarray(mask)+    flattened = _flatsequence(_flatmask(mask))+    return np.array([_ for _ in flattened], dtype=bool)+++def _check_mask_axis(mask, axis, keepdims=np._NoValue):+    "Check whether there are masked values along the given axis"+    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}+    if mask is not nomask:+        return mask.all(axis=axis, **kwargs)+    return nomask+++###############################################################################+#                             Masking functions                               #+###############################################################################++def masked_where(condition, a, copy=True):+    """+    Mask an array where a condition is met.++    Return `a` as an array masked where `condition` is True.+    Any masked values of `a` or `condition` are also masked in the output.++    Parameters+    ----------+    condition : array_like+        Masking condition.  When `condition` tests floating point values for+        equality, consider using ``masked_values`` instead.+    a : array_like+        Array to mask.+    copy : bool+        If True (default) make a copy of `a` in the result.  If False modify+        `a` in place and return a view.++    Returns+    -------+    result : MaskedArray+        The result of masking `a` where `condition` is True.++    See Also+    --------+    masked_values : Mask using floating point equality.+    masked_equal : Mask where equal to a given value.+    masked_not_equal : Mask where `not` equal to a given value.+    masked_less_equal : Mask where less than or equal to a given value.+    masked_greater_equal : Mask where greater than or equal to a given value.+    masked_less : Mask where less than a given value.+    masked_greater : Mask where greater than a given value.+    masked_inside : Mask inside a given interval.+    masked_outside : Mask outside a given interval.+    masked_invalid : Mask invalid values (NaNs or infs).++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(4)+    >>> a+    array([0, 1, 2, 3])+    >>> ma.masked_where(a <= 2, a)+    masked_array(data = [-- -- -- 3],+          mask = [ True  True  True False],+          fill_value=999999)++    Mask array `b` conditional on `a`.++    >>> b = ['a', 'b', 'c', 'd']+    >>> ma.masked_where(a == 2, b)+    masked_array(data = [a b -- d],+          mask = [False False  True False],+          fill_value=N/A)++    Effect of the `copy` argument.++    >>> c = ma.masked_where(a <= 2, a)+    >>> c+    masked_array(data = [-- -- -- 3],+          mask = [ True  True  True False],+          fill_value=999999)+    >>> c[0] = 99+    >>> c+    masked_array(data = [99 -- -- 3],+          mask = [False  True  True False],+          fill_value=999999)+    >>> a+    array([0, 1, 2, 3])+    >>> c = ma.masked_where(a <= 2, a, copy=False)+    >>> c[0] = 99+    >>> c+    masked_array(data = [99 -- -- 3],+          mask = [False  True  True False],+          fill_value=999999)+    >>> a+    array([99,  1,  2,  3])++    When `condition` or `a` contain masked values.++    >>> a = np.arange(4)+    >>> a = ma.masked_where(a == 2, a)+    >>> a+    masked_array(data = [0 1 -- 3],+          mask = [False False  True False],+          fill_value=999999)+    >>> b = np.arange(4)+    >>> b = ma.masked_where(b == 0, b)+    >>> b+    masked_array(data = [-- 1 2 3],+          mask = [ True False False False],+          fill_value=999999)+    >>> ma.masked_where(a == 3, b)+    masked_array(data = [-- 1 -- --],+          mask = [ True False  True  True],+          fill_value=999999)++    """+    # Make sure that condition is a valid standard-type mask.+    cond = make_mask(condition, shrink=False)+    a = np.array(a, copy=copy, subok=True)++    (cshape, ashape) = (cond.shape, a.shape)+    if cshape and cshape != ashape:+        raise IndexError("Inconsistent shape between the condition and the input"+                         " (got %s and %s)" % (cshape, ashape))+    if hasattr(a, '_mask'):+        cond = mask_or(cond, a._mask)+        cls = type(a)+    else:+        cls = MaskedArray+    result = a.view(cls)+    # Assign to *.mask so that structured masks are handled correctly.+    result.mask = _shrink_mask(cond)+    return result+++def masked_greater(x, value, copy=True):+    """+    Mask an array where greater than a given value.++    This function is a shortcut to ``masked_where``, with+    `condition` = (x > value).++    See Also+    --------+    masked_where : Mask where a condition is met.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(4)+    >>> a+    array([0, 1, 2, 3])+    >>> ma.masked_greater(a, 2)+    masked_array(data = [0 1 2 --],+          mask = [False False False  True],+          fill_value=999999)++    """+    return masked_where(greater(x, value), x, copy=copy)+++def masked_greater_equal(x, value, copy=True):+    """+    Mask an array where greater than or equal to a given value.++    This function is a shortcut to ``masked_where``, with+    `condition` = (x >= value).++    See Also+    --------+    masked_where : Mask where a condition is met.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(4)+    >>> a+    array([0, 1, 2, 3])+    >>> ma.masked_greater_equal(a, 2)+    masked_array(data = [0 1 -- --],+          mask = [False False  True  True],+          fill_value=999999)++    """+    return masked_where(greater_equal(x, value), x, copy=copy)+++def masked_less(x, value, copy=True):+    """+    Mask an array where less than a given value.++    This function is a shortcut to ``masked_where``, with+    `condition` = (x < value).++    See Also+    --------+    masked_where : Mask where a condition is met.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(4)+    >>> a+    array([0, 1, 2, 3])+    >>> ma.masked_less(a, 2)+    masked_array(data = [-- -- 2 3],+          mask = [ True  True False False],+          fill_value=999999)++    """+    return masked_where(less(x, value), x, copy=copy)+++def masked_less_equal(x, value, copy=True):+    """+    Mask an array where less than or equal to a given value.++    This function is a shortcut to ``masked_where``, with+    `condition` = (x <= value).++    See Also+    --------+    masked_where : Mask where a condition is met.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(4)+    >>> a+    array([0, 1, 2, 3])+    >>> ma.masked_less_equal(a, 2)+    masked_array(data = [-- -- -- 3],+          mask = [ True  True  True False],+          fill_value=999999)++    """+    return masked_where(less_equal(x, value), x, copy=copy)+++def masked_not_equal(x, value, copy=True):+    """+    Mask an array where `not` equal to a given value.++    This function is a shortcut to ``masked_where``, with+    `condition` = (x != value).++    See Also+    --------+    masked_where : Mask where a condition is met.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(4)+    >>> a+    array([0, 1, 2, 3])+    >>> ma.masked_not_equal(a, 2)+    masked_array(data = [-- -- 2 --],+          mask = [ True  True False  True],+          fill_value=999999)++    """+    return masked_where(not_equal(x, value), x, copy=copy)+++def masked_equal(x, value, copy=True):+    """+    Mask an array where equal to a given value.++    This function is a shortcut to ``masked_where``, with+    `condition` = (x == value).  For floating point arrays,+    consider using ``masked_values(x, value)``.++    See Also+    --------+    masked_where : Mask where a condition is met.+    masked_values : Mask using floating point equality.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(4)+    >>> a+    array([0, 1, 2, 3])+    >>> ma.masked_equal(a, 2)+    masked_array(data = [0 1 -- 3],+          mask = [False False  True False],+          fill_value=999999)++    """+    output = masked_where(equal(x, value), x, copy=copy)+    output.fill_value = value+    return output+++def masked_inside(x, v1, v2, copy=True):+    """+    Mask an array inside a given interval.++    Shortcut to ``masked_where``, where `condition` is True for `x` inside+    the interval [v1,v2] (v1 <= x <= v2).  The boundaries `v1` and `v2`+    can be given in either order.++    See Also+    --------+    masked_where : Mask where a condition is met.++    Notes+    -----+    The array `x` is prefilled with its filling value.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1]+    >>> ma.masked_inside(x, -0.3, 0.3)+    masked_array(data = [0.31 1.2 -- -- -0.4 -1.1],+          mask = [False False  True  True False False],+          fill_value=1e+20)++    The order of `v1` and `v2` doesn't matter.++    >>> ma.masked_inside(x, 0.3, -0.3)+    masked_array(data = [0.31 1.2 -- -- -0.4 -1.1],+          mask = [False False  True  True False False],+          fill_value=1e+20)++    """+    if v2 < v1:+        (v1, v2) = (v2, v1)+    xf = filled(x)+    condition = (xf >= v1) & (xf <= v2)+    return masked_where(condition, x, copy=copy)+++def masked_outside(x, v1, v2, copy=True):+    """+    Mask an array outside a given interval.++    Shortcut to ``masked_where``, where `condition` is True for `x` outside+    the interval [v1,v2] (x < v1)|(x > v2).+    The boundaries `v1` and `v2` can be given in either order.++    See Also+    --------+    masked_where : Mask where a condition is met.++    Notes+    -----+    The array `x` is prefilled with its filling value.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1]+    >>> ma.masked_outside(x, -0.3, 0.3)+    masked_array(data = [-- -- 0.01 0.2 -- --],+          mask = [ True  True False False  True  True],+          fill_value=1e+20)++    The order of `v1` and `v2` doesn't matter.++    >>> ma.masked_outside(x, 0.3, -0.3)+    masked_array(data = [-- -- 0.01 0.2 -- --],+          mask = [ True  True False False  True  True],+          fill_value=1e+20)++    """+    if v2 < v1:+        (v1, v2) = (v2, v1)+    xf = filled(x)+    condition = (xf < v1) | (xf > v2)+    return masked_where(condition, x, copy=copy)+++def masked_object(x, value, copy=True, shrink=True):+    """+    Mask the array `x` where the data are exactly equal to value.++    This function is similar to `masked_values`, but only suitable+    for object arrays: for floating point, use `masked_values` instead.++    Parameters+    ----------+    x : array_like+        Array to mask+    value : object+        Comparison value+    copy : {True, False}, optional+        Whether to return a copy of `x`.+    shrink : {True, False}, optional+        Whether to collapse a mask full of False to nomask++    Returns+    -------+    result : MaskedArray+        The result of masking `x` where equal to `value`.++    See Also+    --------+    masked_where : Mask where a condition is met.+    masked_equal : Mask where equal to a given value (integers).+    masked_values : Mask using floating point equality.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> food = np.array(['green_eggs', 'ham'], dtype=object)+    >>> # don't eat spoiled food+    >>> eat = ma.masked_object(food, 'green_eggs')+    >>> print(eat)+    [-- ham]+    >>> # plain ol` ham is boring+    >>> fresh_food = np.array(['cheese', 'ham', 'pineapple'], dtype=object)+    >>> eat = ma.masked_object(fresh_food, 'green_eggs')+    >>> print(eat)+    [cheese ham pineapple]++    Note that `mask` is set to ``nomask`` if possible.++    >>> eat+    masked_array(data = [cheese ham pineapple],+          mask = False,+          fill_value=?)++    """+    if isMaskedArray(x):+        condition = umath.equal(x._data, value)+        mask = x._mask+    else:+        condition = umath.equal(np.asarray(x), value)+        mask = nomask+    mask = mask_or(mask, make_mask(condition, shrink=shrink))+    return masked_array(x, mask=mask, copy=copy, fill_value=value)+++def masked_values(x, value, rtol=1e-5, atol=1e-8, copy=True, shrink=True):+    """+    Mask using floating point equality.++    Return a MaskedArray, masked where the data in array `x` are approximately+    equal to `value`, determined using `isclose`. The default tolerances for+    `masked_values` are the same as those for `isclose`.++    For integer types, exact equality is used, in the same way as+    `masked_equal`.++    The fill_value is set to `value` and the mask is set to ``nomask`` if+    possible.++    Parameters+    ----------+    x : array_like+        Array to mask.+    value : float+        Masking value.+    rtol, atol : float, optional+        Tolerance parameters passed on to `isclose`+    copy : bool, optional+        Whether to return a copy of `x`.+    shrink : bool, optional+        Whether to collapse a mask full of False to ``nomask``.++    Returns+    -------+    result : MaskedArray+        The result of masking `x` where approximately equal to `value`.++    See Also+    --------+    masked_where : Mask where a condition is met.+    masked_equal : Mask where equal to a given value (integers).++    Examples+    --------+    >>> import numpy.ma as ma+    >>> x = np.array([1, 1.1, 2, 1.1, 3])+    >>> ma.masked_values(x, 1.1)+    masked_array(data = [1.0 -- 2.0 -- 3.0],+          mask = [False  True False  True False],+          fill_value=1.1)++    Note that `mask` is set to ``nomask`` if possible.++    >>> ma.masked_values(x, 1.5)+    masked_array(data = [ 1.   1.1  2.   1.1  3. ],+          mask = False,+          fill_value=1.5)++    For integers, the fill value will be different in general to the+    result of ``masked_equal``.++    >>> x = np.arange(5)+    >>> x+    array([0, 1, 2, 3, 4])+    >>> ma.masked_values(x, 2)+    masked_array(data = [0 1 -- 3 4],+          mask = [False False  True False False],+          fill_value=2)+    >>> ma.masked_equal(x, 2)+    masked_array(data = [0 1 -- 3 4],+          mask = [False False  True False False],+          fill_value=999999)++    """+    xnew = filled(x, value)+    if np.issubdtype(xnew.dtype, np.floating):+        mask = np.isclose(xnew, value, atol=atol, rtol=rtol)+    else:+        mask = umath.equal(xnew, value)+    ret = masked_array(xnew, mask=mask, copy=copy, fill_value=value)+    if shrink:+        ret.shrink_mask()+    return ret+++def masked_invalid(a, copy=True):+    """+    Mask an array where invalid values occur (NaNs or infs).++    This function is a shortcut to ``masked_where``, with+    `condition` = ~(np.isfinite(a)). Any pre-existing mask is conserved.+    Only applies to arrays with a dtype where NaNs or infs make sense+    (i.e. floating point types), but accepts any array_like object.++    See Also+    --------+    masked_where : Mask where a condition is met.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.arange(5, dtype=float)+    >>> a[2] = np.NaN+    >>> a[3] = np.PINF+    >>> a+    array([  0.,   1.,  NaN,  Inf,   4.])+    >>> ma.masked_invalid(a)+    masked_array(data = [0.0 1.0 -- -- 4.0],+          mask = [False False  True  True False],+          fill_value=1e+20)++    """+    a = np.array(a, copy=copy, subok=True)+    mask = getattr(a, '_mask', None)+    if mask is not None:+        condition = ~(np.isfinite(getdata(a)))+        if mask is not nomask:+            condition |= mask+        cls = type(a)+    else:+        condition = ~(np.isfinite(a))+        cls = MaskedArray+    result = a.view(cls)+    result._mask = condition+    return result+++###############################################################################+#                            Printing options                                 #+###############################################################################+++class _MaskedPrintOption(object):+    """+    Handle the string used to represent missing data in a masked array.++    """++    def __init__(self, display):+        """+        Create the masked_print_option object.++        """+        self._display = display+        self._enabled = True++    def display(self):+        """+        Display the string to print for masked values.++        """+        return self._display++    def set_display(self, s):+        """+        Set the string to print for masked values.++        """+        self._display = s++    def enabled(self):+        """+        Is the use of the display value enabled?++        """+        return self._enabled++    def enable(self, shrink=1):+        """+        Set the enabling shrink to `shrink`.++        """+        self._enabled = shrink++    def __str__(self):+        return str(self._display)++    __repr__ = __str__++# if you single index into a masked location you get this object.+masked_print_option = _MaskedPrintOption('--')+++def _recursive_printoption(result, mask, printopt):+    """+    Puts printoptions in result where mask is True.++    Private function allowing for recursion++    """+    names = result.dtype.names+    if names is not None:+        for name in names:+            curdata = result[name]+            curmask = mask[name]+            _recursive_printoption(curdata, curmask, printopt)+    else:+        np.copyto(result, printopt, where=mask)+    return++# For better or worse, these end in a newline+_legacy_print_templates = dict(+    long_std=textwrap.dedent("""\+        masked_%(name)s(data =+         %(data)s,+        %(nlen)s        mask =+         %(mask)s,+        %(nlen)s  fill_value = %(fill)s)+        """),+    long_flx=textwrap.dedent("""\+        masked_%(name)s(data =+         %(data)s,+        %(nlen)s        mask =+         %(mask)s,+        %(nlen)s  fill_value = %(fill)s,+        %(nlen)s       dtype = %(dtype)s)+        """),+    short_std=textwrap.dedent("""\+        masked_%(name)s(data = %(data)s,+        %(nlen)s        mask = %(mask)s,+        %(nlen)s  fill_value = %(fill)s)+        """),+    short_flx=textwrap.dedent("""\+        masked_%(name)s(data = %(data)s,+        %(nlen)s        mask = %(mask)s,+        %(nlen)s  fill_value = %(fill)s,+        %(nlen)s       dtype = %(dtype)s)+        """)+)++###############################################################################+#                          MaskedArray class                                  #+###############################################################################+++def _recursive_filled(a, mask, fill_value):+    """+    Recursively fill `a` with `fill_value`.++    """+    names = a.dtype.names+    for name in names:+        current = a[name]+        if current.dtype.names is not None:+            _recursive_filled(current, mask[name], fill_value[name])+        else:+            np.copyto(current, fill_value[name], where=mask[name])+++def flatten_structured_array(a):+    """+    Flatten a structured array.++    The data type of the output is chosen such that it can represent all of the+    (nested) fields.++    Parameters+    ----------+    a : structured array++    Returns+    -------+    output : masked array or ndarray+        A flattened masked array if the input is a masked array, otherwise a+        standard ndarray.++    Examples+    --------+    >>> ndtype = [('a', int), ('b', float)]+    >>> a = np.array([(1, 1), (2, 2)], dtype=ndtype)+    >>> flatten_structured_array(a)+    array([[1., 1.],+           [2., 2.]])++    """++    def flatten_sequence(iterable):+        """+        Flattens a compound of nested iterables.++        """+        for elm in iter(iterable):+            if hasattr(elm, '__iter__'):+                for f in flatten_sequence(elm):+                    yield f+            else:+                yield elm++    a = np.asanyarray(a)+    inishape = a.shape+    a = a.ravel()+    if isinstance(a, MaskedArray):+        out = np.array([tuple(flatten_sequence(d.item())) for d in a._data])+        out = out.view(MaskedArray)+        out._mask = np.array([tuple(flatten_sequence(d.item()))+                              for d in getmaskarray(a)])+    else:+        out = np.array([tuple(flatten_sequence(d.item())) for d in a])+    if len(inishape) > 1:+        newshape = list(out.shape)+        newshape[0] = inishape+        out.shape = tuple(flatten_sequence(newshape))+    return out+++def _arraymethod(funcname, onmask=True):+    """+    Return a class method wrapper around a basic array method.++    Creates a class method which returns a masked array, where the new+    ``_data`` array is the output of the corresponding basic method called+    on the original ``_data``.++    If `onmask` is True, the new mask is the output of the method called+    on the initial mask. Otherwise, the new mask is just a reference+    to the initial mask.++    Parameters+    ----------+    funcname : str+        Name of the function to apply on data.+    onmask : bool+        Whether the mask must be processed also (True) or left+        alone (False). Default is True. Make available as `_onmask`+        attribute.++    Returns+    -------+    method : instancemethod+        Class method wrapper of the specified basic array method.++    """+    def wrapped_method(self, *args, **params):+        result = getattr(self._data, funcname)(*args, **params)+        result = result.view(type(self))+        result._update_from(self)+        mask = self._mask+        if not onmask:+            result.__setmask__(mask)+        elif mask is not nomask:+            # __setmask__ makes a copy, which we don't want+            result._mask = getattr(mask, funcname)(*args, **params)+        return result+    methdoc = getattr(ndarray, funcname, None) or getattr(np, funcname, None)+    if methdoc is not None:+        wrapped_method.__doc__ = methdoc.__doc__+    wrapped_method.__name__ = funcname+    return wrapped_method+++class MaskedIterator(object):+    """+    Flat iterator object to iterate over masked arrays.++    A `MaskedIterator` iterator is returned by ``x.flat`` for any masked array+    `x`. It allows iterating over the array as if it were a 1-D array,+    either in a for-loop or by calling its `next` method.++    Iteration is done in C-contiguous style, with the last index varying the+    fastest. The iterator can also be indexed using basic slicing or+    advanced indexing.++    See Also+    --------+    MaskedArray.flat : Return a flat iterator over an array.+    MaskedArray.flatten : Returns a flattened copy of an array.++    Notes+    -----+    `MaskedIterator` is not exported by the `ma` module. Instead of+    instantiating a `MaskedIterator` directly, use `MaskedArray.flat`.++    Examples+    --------+    >>> x = np.ma.array(arange(6).reshape(2, 3))+    >>> fl = x.flat+    >>> type(fl)+    <class 'numpy.ma.core.MaskedIterator'>+    >>> for item in fl:+    ...     print(item)+    ...+    0+    1+    2+    3+    4+    5++    Extracting more than a single element b indexing the `MaskedIterator`+    returns a masked array:++    >>> fl[2:4]+    masked_array(data = [2 3],+                 mask = False,+           fill_value = 999999)++    """++    def __init__(self, ma):+        self.ma = ma+        self.dataiter = ma._data.flat++        if ma._mask is nomask:+            self.maskiter = None+        else:+            self.maskiter = ma._mask.flat++    def __iter__(self):+        return self++    def __getitem__(self, indx):+        result = self.dataiter.__getitem__(indx).view(type(self.ma))+        if self.maskiter is not None:+            _mask = self.maskiter.__getitem__(indx)+            if isinstance(_mask, ndarray):+                # set shape to match that of data; this is needed for matrices+                _mask.shape = result.shape+                result._mask = _mask+            elif isinstance(_mask, np.void):+                return mvoid(result, mask=_mask, hardmask=self.ma._hardmask)+            elif _mask:  # Just a scalar, masked+                return masked+        return result++    # This won't work if ravel makes a copy+    def __setitem__(self, index, value):+        self.dataiter[index] = getdata(value)+        if self.maskiter is not None:+            self.maskiter[index] = getmaskarray(value)++    def __next__(self):+        """+        Return the next value, or raise StopIteration.++        Examples+        --------+        >>> x = np.ma.array([3, 2], mask=[0, 1])+        >>> fl = x.flat+        >>> fl.next()+        3+        >>> fl.next()+        masked_array(data = --,+                     mask = True,+               fill_value = 1e+20)+        >>> fl.next()+        Traceback (most recent call last):+          File "<stdin>", line 1, in <module>+          File "/home/ralf/python/numpy/numpy/ma/core.py", line 2243, in next+            d = self.dataiter.next()+        StopIteration++        """+        d = next(self.dataiter)+        if self.maskiter is not None:+            m = next(self.maskiter)+            if isinstance(m, np.void):+                return mvoid(d, mask=m, hardmask=self.ma._hardmask)+            elif m:  # Just a scalar, masked+                return masked+        return d++    next = __next__+++class MaskedArray(ndarray):+    """+    An array class with possibly masked values.++    Masked values of True exclude the corresponding element from any+    computation.++    Construction::++      x = MaskedArray(data, mask=nomask, dtype=None, copy=False, subok=True,+                      ndmin=0, fill_value=None, keep_mask=True, hard_mask=None,+                      shrink=True, order=None)++    Parameters+    ----------+    data : array_like+        Input data.+    mask : sequence, optional+        Mask. Must be convertible to an array of booleans with the same+        shape as `data`. True indicates a masked (i.e. invalid) data.+    dtype : dtype, optional+        Data type of the output.+        If `dtype` is None, the type of the data argument (``data.dtype``)+        is used. If `dtype` is not None and different from ``data.dtype``,+        a copy is performed.+    copy : bool, optional+        Whether to copy the input data (True), or to use a reference instead.+        Default is False.+    subok : bool, optional+        Whether to return a subclass of `MaskedArray` if possible (True) or a+        plain `MaskedArray`. Default is True.+    ndmin : int, optional+        Minimum number of dimensions. Default is 0.+    fill_value : scalar, optional+        Value used to fill in the masked values when necessary.+        If None, a default based on the data-type is used.+    keep_mask : bool, optional+        Whether to combine `mask` with the mask of the input data, if any+        (True), or to use only `mask` for the output (False). Default is True.+    hard_mask : bool, optional+        Whether to use a hard mask or not. With a hard mask, masked values+        cannot be unmasked. Default is False.+    shrink : bool, optional+        Whether to force compression of an empty mask. Default is True.+    order : {'C', 'F', 'A'}, optional+        Specify the order of the array.  If order is 'C', then the array+        will be in C-contiguous order (last-index varies the fastest).+        If order is 'F', then the returned array will be in+        Fortran-contiguous order (first-index varies the fastest).+        If order is 'A' (default), then the returned array may be+        in any order (either C-, Fortran-contiguous, or even discontiguous),+        unless a copy is required, in which case it will be C-contiguous.++    """++    __array_priority__ = 15+    _defaultmask = nomask+    _defaulthardmask = False+    _baseclass = ndarray++    # Maximum number of elements per axis used when printing an array. The+    # 1d case is handled separately because we need more values in this case.+    _print_width = 100+    _print_width_1d = 1500++    def __new__(cls, data=None, mask=nomask, dtype=None, copy=False,+                subok=True, ndmin=0, fill_value=None, keep_mask=True,+                hard_mask=None, shrink=True, order=None, **options):+        """+        Create a new masked array from scratch.++        Notes+        -----+        A masked array can also be created by taking a .view(MaskedArray).++        """+        # Process data.+        _data = np.array(data, dtype=dtype, copy=copy,+                         order=order, subok=True, ndmin=ndmin)+        _baseclass = getattr(data, '_baseclass', type(_data))+        # Check that we're not erasing the mask.+        if isinstance(data, MaskedArray) and (data.shape != _data.shape):+            copy = True++        # Here, we copy the _view_, so that we can attach new properties to it+        # we must never do .view(MaskedConstant), as that would create a new+        # instance of np.ma.masked, which make identity comparison fail+        if isinstance(data, cls) and subok and not isinstance(data, MaskedConstant):+            _data = ndarray.view(_data, type(data))+        else:+            _data = ndarray.view(_data, cls)+        # Backwards compatibility w/ numpy.core.ma.+        if hasattr(data, '_mask') and not isinstance(data, ndarray):+            _data._mask = data._mask+            # FIXME _sharedmask is never used.+            _sharedmask = True+        # Process mask.+        # Type of the mask+        mdtype = make_mask_descr(_data.dtype)++        if mask is nomask:+            # Case 1. : no mask in input.+            # Erase the current mask ?+            if not keep_mask:+                # With a reduced version+                if shrink:+                    _data._mask = nomask+                # With full version+                else:+                    _data._mask = np.zeros(_data.shape, dtype=mdtype)+            # Check whether we missed something+            elif isinstance(data, (tuple, list)):+                try:+                    # If data is a sequence of masked array+                    mask = np.array([getmaskarray(m) for m in data],+                                    dtype=mdtype)+                except ValueError:+                    # If data is nested+                    mask = nomask+                # Force shrinking of the mask if needed (and possible)+                if (mdtype == MaskType) and mask.any():+                    _data._mask = mask+                    _data._sharedmask = False+            else:+                _data._sharedmask = not copy+                if copy:+                    _data._mask = _data._mask.copy()+                    # Reset the shape of the original mask+                    if getmask(data) is not nomask:+                        data._mask.shape = data.shape+        else:+            # Case 2. : With a mask in input.+            # If mask is boolean, create an array of True or False+            if mask is True and mdtype == MaskType:+                mask = np.ones(_data.shape, dtype=mdtype)+            elif mask is False and mdtype == MaskType:+                mask = np.zeros(_data.shape, dtype=mdtype)+            else:+                # Read the mask with the current mdtype+                try:+                    mask = np.array(mask, copy=copy, dtype=mdtype)+                # Or assume it's a sequence of bool/int+                except TypeError:+                    mask = np.array([tuple([m] * len(mdtype)) for m in mask],+                                    dtype=mdtype)+            # Make sure the mask and the data have the same shape+            if mask.shape != _data.shape:+                (nd, nm) = (_data.size, mask.size)+                if nm == 1:+                    mask = np.resize(mask, _data.shape)+                elif nm == nd:+                    mask = np.reshape(mask, _data.shape)+                else:+                    msg = "Mask and data not compatible: data size is %i, " + \+                          "mask size is %i."+                    raise MaskError(msg % (nd, nm))+                copy = True+            # Set the mask to the new value+            if _data._mask is nomask:+                _data._mask = mask+                _data._sharedmask = not copy+            else:+                if not keep_mask:+                    _data._mask = mask+                    _data._sharedmask = not copy+                else:+                    if _data.dtype.names is not None:+                        def _recursive_or(a, b):+                            "do a|=b on each field of a, recursively"+                            for name in a.dtype.names:+                                (af, bf) = (a[name], b[name])+                                if af.dtype.names is not None:+                                    _recursive_or(af, bf)+                                else:+                                    af |= bf++                        _recursive_or(_data._mask, mask)+                    else:+                        _data._mask = np.logical_or(mask, _data._mask)+                    _data._sharedmask = False+        # Update fill_value.+        if fill_value is None:+            fill_value = getattr(data, '_fill_value', None)+        # But don't run the check unless we have something to check.+        if fill_value is not None:+            _data._fill_value = _check_fill_value(fill_value, _data.dtype)+        # Process extra options ..+        if hard_mask is None:+            _data._hardmask = getattr(data, '_hardmask', False)+        else:+            _data._hardmask = hard_mask+        _data._baseclass = _baseclass+        return _data+++    def _update_from(self, obj):+        """+        Copies some attributes of obj to self.++        """+        if isinstance(obj, ndarray):+            _baseclass = type(obj)+        else:+            _baseclass = ndarray+        # We need to copy the _basedict to avoid backward propagation+        _optinfo = {}+        _optinfo.update(getattr(obj, '_optinfo', {}))+        _optinfo.update(getattr(obj, '_basedict', {}))+        if not isinstance(obj, MaskedArray):+            _optinfo.update(getattr(obj, '__dict__', {}))+        _dict = dict(_fill_value=getattr(obj, '_fill_value', None),+                     _hardmask=getattr(obj, '_hardmask', False),+                     _sharedmask=getattr(obj, '_sharedmask', False),+                     _isfield=getattr(obj, '_isfield', False),+                     _baseclass=getattr(obj, '_baseclass', _baseclass),+                     _optinfo=_optinfo,+                     _basedict=_optinfo)+        self.__dict__.update(_dict)+        self.__dict__.update(_optinfo)+        return++    def __array_finalize__(self, obj):+        """+        Finalizes the masked array.++        """+        # Get main attributes.+        self._update_from(obj)++        # We have to decide how to initialize self.mask, based on+        # obj.mask. This is very difficult.  There might be some+        # correspondence between the elements in the array we are being+        # created from (= obj) and us. Or there might not. This method can+        # be called in all kinds of places for all kinds of reasons -- could+        # be empty_like, could be slicing, could be a ufunc, could be a view.+        # The numpy subclassing interface simply doesn't give us any way+        # to know, which means that at best this method will be based on+        # guesswork and heuristics. To make things worse, there isn't even any+        # clear consensus about what the desired behavior is. For instance,+        # most users think that np.empty_like(marr) -- which goes via this+        # method -- should return a masked array with an empty mask (see+        # gh-3404 and linked discussions), but others disagree, and they have+        # existing code which depends on empty_like returning an array that+        # matches the input mask.+        #+        # Historically our algorithm was: if the template object mask had the+        # same *number of elements* as us, then we used *it's mask object+        # itself* as our mask, so that writes to us would also write to the+        # original array. This is horribly broken in multiple ways.+        #+        # Now what we do instead is, if the template object mask has the same+        # number of elements as us, and we do not have the same base pointer+        # as the template object (b/c views like arr[...] should keep the same+        # mask), then we make a copy of the template object mask and use+        # that. This is also horribly broken but somewhat less so. Maybe.+        if isinstance(obj, ndarray):+            # XX: This looks like a bug -- shouldn't it check self.dtype+            # instead?+            if obj.dtype.names is not None:+                _mask = getmaskarray(obj)+            else:+                _mask = getmask(obj)++            # If self and obj point to exactly the same data, then probably+            # self is a simple view of obj (e.g., self = obj[...]), so they+            # should share the same mask. (This isn't 100% reliable, e.g. self+            # could be the first row of obj, or have strange strides, but as a+            # heuristic it's not bad.) In all other cases, we make a copy of+            # the mask, so that future modifications to 'self' do not end up+            # side-effecting 'obj' as well.+            if (_mask is not nomask and obj.__array_interface__["data"][0]+                    != self.__array_interface__["data"][0]):+                # We should make a copy. But we could get here via astype,+                # in which case the mask might need a new dtype as well+                # (e.g., changing to or from a structured dtype), and the+                # order could have changed. So, change the mask type if+                # needed and use astype instead of copy.+                if self.dtype == obj.dtype:+                    _mask_dtype = _mask.dtype+                else:+                    _mask_dtype = make_mask_descr(self.dtype)++                if self.flags.c_contiguous:+                    order = "C"+                elif self.flags.f_contiguous:+                    order = "F"+                else:+                    order = "K"++                _mask = _mask.astype(_mask_dtype, order)+            else:+                # Take a view so shape changes, etc., do not propagate back.+                _mask = _mask.view()+        else:+            _mask = nomask++        self._mask = _mask+        # Finalize the mask+        if self._mask is not nomask:+            try:+                self._mask.shape = self.shape+            except ValueError:+                self._mask = nomask+            except (TypeError, AttributeError):+                # When _mask.shape is not writable (because it's a void)+                pass+        # Finalize the fill_value for structured arrays+        if self.dtype.names is not None:+            if self._fill_value is None:+                self._fill_value = _check_fill_value(None, self.dtype)+        return++    def __array_wrap__(self, obj, context=None):+        """+        Special hook for ufuncs.++        Wraps the numpy array and sets the mask according to context.++        """+        if obj is self:  # for in-place operations+            result = obj+        else:+            result = obj.view(type(self))+            result._update_from(self)++        if context is not None:+            result._mask = result._mask.copy()+            func, args, out_i = context+            # args sometimes contains outputs (gh-10459), which we don't want+            input_args = args[:func.nin]+            m = reduce(mask_or, [getmaskarray(arg) for arg in input_args])+            # Get the domain mask+            domain = ufunc_domain.get(func, None)+            if domain is not None:+                # Take the domain, and make sure it's a ndarray+                with np.errstate(divide='ignore', invalid='ignore'):+                    d = filled(domain(*input_args), True)++                if d.any():+                    # Fill the result where the domain is wrong+                    try:+                        # Binary domain: take the last value+                        fill_value = ufunc_fills[func][-1]+                    except TypeError:+                        # Unary domain: just use this one+                        fill_value = ufunc_fills[func]+                    except KeyError:+                        # Domain not recognized, use fill_value instead+                        fill_value = self.fill_value++                    np.copyto(result, fill_value, where=d)++                    # Update the mask+                    if m is nomask:+                        m = d+                    else:+                        # Don't modify inplace, we risk back-propagation+                        m = (m | d)++            # Make sure the mask has the proper size+            if result is not self and result.shape == () and m:+                return masked+            else:+                result._mask = m+                result._sharedmask = False++        return result++    def view(self, dtype=None, type=None, fill_value=None):+        """+        Return a view of the MaskedArray data++        Parameters+        ----------+        dtype : data-type or ndarray sub-class, optional+            Data-type descriptor of the returned view, e.g., float32 or int16.+            The default, None, results in the view having the same data-type+            as `a`. As with ``ndarray.view``, dtype can also be specified as+            an ndarray sub-class, which then specifies the type of the+            returned object (this is equivalent to setting the ``type``+            parameter).+        type : Python type, optional+            Type of the returned view, either ndarray or a subclass.  The+            default None results in type preservation.++        Notes+        -----++        ``a.view()`` is used two different ways:++        ``a.view(some_dtype)`` or ``a.view(dtype=some_dtype)`` constructs a view+        of the array's memory with a different data-type.  This can cause a+        reinterpretation of the bytes of memory.++        ``a.view(ndarray_subclass)`` or ``a.view(type=ndarray_subclass)`` just+        returns an instance of `ndarray_subclass` that looks at the same array+        (same shape, dtype, etc.)  This does not cause a reinterpretation of the+        memory.++        If `fill_value` is not specified, but `dtype` is specified (and is not+        an ndarray sub-class), the `fill_value` of the MaskedArray will be+        reset. If neither `fill_value` nor `dtype` are specified (or if+        `dtype` is an ndarray sub-class), then the fill value is preserved.+        Finally, if `fill_value` is specified, but `dtype` is not, the fill+        value is set to the specified value.++        For ``a.view(some_dtype)``, if ``some_dtype`` has a different number of+        bytes per entry than the previous dtype (for example, converting a+        regular array to a structured array), then the behavior of the view+        cannot be predicted just from the superficial appearance of ``a`` (shown+        by ``print(a)``). It also depends on exactly how ``a`` is stored in+        memory. Therefore if ``a`` is C-ordered versus fortran-ordered, versus+        defined as a slice or transpose, etc., the view may give different+        results.+        """++        if dtype is None:+            if type is None:+                output = ndarray.view(self)+            else:+                output = ndarray.view(self, type)+        elif type is None:+            try:+                if issubclass(dtype, ndarray):+                    output = ndarray.view(self, dtype)+                    dtype = None+                else:+                    output = ndarray.view(self, dtype)+            except TypeError:+                output = ndarray.view(self, dtype)+        else:+            output = ndarray.view(self, dtype, type)++        # also make the mask be a view (so attr changes to the view's+        # mask do no affect original object's mask)+        # (especially important to avoid affecting np.masked singleton)+        if (getmask(output) is not nomask):+            output._mask = output._mask.view()++        # Make sure to reset the _fill_value if needed+        if getattr(output, '_fill_value', None) is not None:+            if fill_value is None:+                if dtype is None:+                    pass  # leave _fill_value as is+                else:+                    output._fill_value = None+            else:+                output.fill_value = fill_value+        return output+    view.__doc__ = ndarray.view.__doc__++    def __getitem__(self, indx):+        """+        x.__getitem__(y) <==> x[y]++        Return the item described by i, as a masked array.++        """+        # We could directly use ndarray.__getitem__ on self.+        # But then we would have to modify __array_finalize__ to prevent the+        # mask of being reshaped if it hasn't been set up properly yet+        # So it's easier to stick to the current version+        dout = self.data[indx]+        _mask = self._mask++        def _is_scalar(m):+            return not isinstance(m, np.ndarray)++        def _scalar_heuristic(arr, elem):+            """+            Return whether `elem` is a scalar result of indexing `arr`, or None+            if undecidable without promoting nomask to a full mask+            """+            # obviously a scalar+            if not isinstance(elem, np.ndarray):+                return True++            # object array scalar indexing can return anything+            elif arr.dtype.type is np.object_:+                if arr.dtype is not elem.dtype:+                    # elem is an array, but dtypes do not match, so must be+                    # an element+                    return True++            # well-behaved subclass that only returns 0d arrays when+            # expected - this is not a scalar+            elif type(arr).__getitem__ == ndarray.__getitem__:+                return False++            return None++        if _mask is not nomask:+            # _mask cannot be a subclass, so it tells us whether we should+            # expect a scalar. It also cannot be of dtype object.+            mout = _mask[indx]+            scalar_expected = _is_scalar(mout)++        else:+            # attempt to apply the heuristic to avoid constructing a full mask+            mout = nomask+            scalar_expected = _scalar_heuristic(self.data, dout)+            if scalar_expected is None:+                # heuristics have failed+                # construct a full array, so we can be certain. This is costly.+                # we could also fall back on ndarray.__getitem__(self.data, indx)+                scalar_expected = _is_scalar(getmaskarray(self)[indx])++        # Did we extract a single item?+        if scalar_expected:+            # A record+            if isinstance(dout, np.void):+                # We should always re-cast to mvoid, otherwise users can+                # change masks on rows that already have masked values, but not+                # on rows that have no masked values, which is inconsistent.+                return mvoid(dout, mask=mout, hardmask=self._hardmask)++            # special case introduced in gh-5962+            elif (self.dtype.type is np.object_ and+                  isinstance(dout, np.ndarray) and+                  dout is not masked):+                # If masked, turn into a MaskedArray, with everything masked.+                if mout:+                    return MaskedArray(dout, mask=True)+                else:+                    return dout++            # Just a scalar+            else:+                if mout:+                    return masked+                else:+                    return dout+        else:+            # Force dout to MA+            dout = dout.view(type(self))+            # Inherit attributes from self+            dout._update_from(self)+            # Check the fill_value+            if isinstance(indx, basestring):+                if self._fill_value is not None:+                    dout._fill_value = self._fill_value[indx]++                    # If we're indexing a multidimensional field in a+                    # structured array (such as dtype("(2,)i2,(2,)i1")),+                    # dimensionality goes up (M[field].ndim == M.ndim ++                    # M.dtype[field].ndim).  That's fine for+                    # M[field] but problematic for M[field].fill_value+                    # which should have shape () to avoid breaking several+                    # methods. There is no great way out, so set to+                    # first element.  See issue #6723.+                    if dout._fill_value.ndim > 0:+                        if not (dout._fill_value ==+                                dout._fill_value.flat[0]).all():+                            warnings.warn(+                                "Upon accessing multidimensional field "+                                "{indx:s}, need to keep dimensionality "+                                "of fill_value at 0. Discarding "+                                "heterogeneous fill_value and setting "+                                "all to {fv!s}.".format(indx=indx,+                                    fv=dout._fill_value[0]),+                                stacklevel=2)+                        dout._fill_value = dout._fill_value.flat[0]+                dout._isfield = True+            # Update the mask if needed+            if mout is not nomask:+                # set shape to match that of data; this is needed for matrices+                dout._mask = reshape(mout, dout.shape)+                dout._sharedmask = True+                # Note: Don't try to check for m.any(), that'll take too long+        return dout++    def __setitem__(self, indx, value):+        """+        x.__setitem__(i, y) <==> x[i]=y++        Set item described by index. If value is masked, masks those+        locations.++        """+        if self is masked:+            raise MaskError('Cannot alter the masked element.')+        _data = self._data+        _mask = self._mask+        if isinstance(indx, basestring):+            _data[indx] = value+            if _mask is nomask:+                self._mask = _mask = make_mask_none(self.shape, self.dtype)+            _mask[indx] = getmask(value)+            return++        _dtype = _data.dtype++        if value is masked:+            # The mask wasn't set: create a full version.+            if _mask is nomask:+                _mask = self._mask = make_mask_none(self.shape, _dtype)+            # Now, set the mask to its value.+            if _dtype.names is not None:+                _mask[indx] = tuple([True] * len(_dtype.names))+            else:+                _mask[indx] = True+            return++        # Get the _data part of the new value+        dval = getattr(value, '_data', value)+        # Get the _mask part of the new value+        mval = getmask(value)+        if _dtype.names is not None and mval is nomask:+            mval = tuple([False] * len(_dtype.names))+        if _mask is nomask:+            # Set the data, then the mask+            _data[indx] = dval+            if mval is not nomask:+                _mask = self._mask = make_mask_none(self.shape, _dtype)+                _mask[indx] = mval+        elif not self._hardmask:+            # Set the data, then the mask+            _data[indx] = dval+            _mask[indx] = mval+        elif hasattr(indx, 'dtype') and (indx.dtype == MaskType):+            indx = indx * umath.logical_not(_mask)+            _data[indx] = dval+        else:+            if _dtype.names is not None:+                err_msg = "Flexible 'hard' masks are not yet supported."+                raise NotImplementedError(err_msg)+            mindx = mask_or(_mask[indx], mval, copy=True)+            dindx = self._data[indx]+            if dindx.size > 1:+                np.copyto(dindx, dval, where=~mindx)+            elif mindx is nomask:+                dindx = dval+            _data[indx] = dindx+            _mask[indx] = mindx+        return++    # Define so that we can overwrite the setter.+    @property+    def dtype(self):+        return super(MaskedArray, self).dtype++    @dtype.setter+    def dtype(self, dtype):+        super(MaskedArray, type(self)).dtype.__set__(self, dtype)+        if self._mask is not nomask:+            self._mask = self._mask.view(make_mask_descr(dtype), ndarray)+            # Try to reset the shape of the mask (if we don't have a void).+            # This raises a ValueError if the dtype change won't work.+            try:+                self._mask.shape = self.shape+            except (AttributeError, TypeError):+                pass++    @property+    def shape(self):+        return super(MaskedArray, self).shape++    @shape.setter+    def shape(self, shape):+        super(MaskedArray, type(self)).shape.__set__(self, shape)+        # Cannot use self._mask, since it may not (yet) exist when a+        # masked matrix sets the shape.+        if getmask(self) is not nomask:+            self._mask.shape = self.shape++    def __setmask__(self, mask, copy=False):+        """+        Set the mask.++        """+        idtype = self.dtype+        current_mask = self._mask+        if mask is masked:+            mask = True++        if (current_mask is nomask):+            # Make sure the mask is set+            # Just don't do anything if there's nothing to do.+            if mask is nomask:+                return+            current_mask = self._mask = make_mask_none(self.shape, idtype)++        if idtype.names is None:+            # No named fields.+            # Hardmask: don't unmask the data+            if self._hardmask:+                current_mask |= mask+            # Softmask: set everything to False+            # If it's obviously a compatible scalar, use a quick update+            # method.+            elif isinstance(mask, (int, float, np.bool_, np.number)):+                current_mask[...] = mask+            # Otherwise fall back to the slower, general purpose way.+            else:+                current_mask.flat = mask+        else:+            # Named fields w/+            mdtype = current_mask.dtype+            mask = np.array(mask, copy=False)+            # Mask is a singleton+            if not mask.ndim:+                # It's a boolean : make a record+                if mask.dtype.kind == 'b':+                    mask = np.array(tuple([mask.item()] * len(mdtype)),+                                    dtype=mdtype)+                # It's a record: make sure the dtype is correct+                else:+                    mask = mask.astype(mdtype)+            # Mask is a sequence+            else:+                # Make sure the new mask is a ndarray with the proper dtype+                try:+                    mask = np.array(mask, copy=copy, dtype=mdtype)+                # Or assume it's a sequence of bool/int+                except TypeError:+                    mask = np.array([tuple([m] * len(mdtype)) for m in mask],+                                    dtype=mdtype)+            # Hardmask: don't unmask the data+            if self._hardmask:+                for n in idtype.names:+                    current_mask[n] |= mask[n]+            # Softmask: set everything to False+            # If it's obviously a compatible scalar, use a quick update+            # method.+            elif isinstance(mask, (int, float, np.bool_, np.number)):+                current_mask[...] = mask+            # Otherwise fall back to the slower, general purpose way.+            else:+                current_mask.flat = mask+        # Reshape if needed+        if current_mask.shape:+            current_mask.shape = self.shape+        return++    _set_mask = __setmask__++    def _get_mask(self):+        """Return the current mask.++        """+        # We could try to force a reshape, but that wouldn't work in some+        # cases.+        return self._mask++    mask = property(fget=_get_mask, fset=__setmask__, doc="Mask")++    def _get_recordmask(self):+        """+        Return the mask of the records.++        A record is masked when all the fields are masked.++        """+        _mask = self._mask.view(ndarray)+        if _mask.dtype.names is None:+            return _mask+        return np.all(flatten_structured_array(_mask), axis=-1)++    def _set_recordmask(self):+        """+        Return the mask of the records.++        A record is masked when all the fields are masked.++        """+        raise NotImplementedError("Coming soon: setting the mask per records!")++    recordmask = property(fget=_get_recordmask)++    def harden_mask(self):+        """+        Force the mask to hard.++        Whether the mask of a masked array is hard or soft is determined by+        its `hardmask` property. `harden_mask` sets `hardmask` to True.++        See Also+        --------+        hardmask++        """+        self._hardmask = True+        return self++    def soften_mask(self):+        """+        Force the mask to soft.++        Whether the mask of a masked array is hard or soft is determined by+        its `hardmask` property. `soften_mask` sets `hardmask` to False.++        See Also+        --------+        hardmask++        """+        self._hardmask = False+        return self++    hardmask = property(fget=lambda self: self._hardmask,+                        doc="Hardness of the mask")++    def unshare_mask(self):+        """+        Copy the mask and set the sharedmask flag to False.++        Whether the mask is shared between masked arrays can be seen from+        the `sharedmask` property. `unshare_mask` ensures the mask is not shared.+        A copy of the mask is only made if it was shared.++        See Also+        --------+        sharedmask++        """+        if self._sharedmask:+            self._mask = self._mask.copy()+            self._sharedmask = False+        return self++    sharedmask = property(fget=lambda self: self._sharedmask,+                          doc="Share status of the mask (read-only).")++    def shrink_mask(self):+        """+        Reduce a mask to nomask when possible.++        Parameters+        ----------+        None++        Returns+        -------+        None++        Examples+        --------+        >>> x = np.ma.array([[1,2 ], [3, 4]], mask=[0]*4)+        >>> x.mask+        array([[False, False],+               [False, False]])+        >>> x.shrink_mask()+        >>> x.mask+        False++        """+        self._mask = _shrink_mask(self._mask)+        return self++    baseclass = property(fget=lambda self: self._baseclass,+                         doc="Class of the underlying data (read-only).")++    def _get_data(self):+        """Return the current data, as a view of the original+        underlying data.++        """+        return ndarray.view(self, self._baseclass)++    _data = property(fget=_get_data)+    data = property(fget=_get_data)++    def _get_flat(self):+        "Return a flat iterator."+        return MaskedIterator(self)++    def _set_flat(self, value):+        "Set a flattened version of self to value."+        y = self.ravel()+        y[:] = value++    flat = property(fget=_get_flat, fset=_set_flat,+                    doc="Flat version of the array.")++    def get_fill_value(self):+        """+        Return the filling value of the masked array.++        Returns+        -------+        fill_value : scalar+            The filling value.++        Examples+        --------+        >>> for dt in [np.int32, np.int64, np.float64, np.complex128]:+        ...     np.ma.array([0, 1], dtype=dt).get_fill_value()+        ...+        999999+        999999+        1e+20+        (1e+20+0j)++        >>> x = np.ma.array([0, 1.], fill_value=-np.inf)+        >>> x.get_fill_value()+        -inf++        """+        if self._fill_value is None:+            self._fill_value = _check_fill_value(None, self.dtype)++        # Temporary workaround to account for the fact that str and bytes+        # scalars cannot be indexed with (), whereas all other numpy+        # scalars can. See issues #7259 and #7267.+        # The if-block can be removed after #7267 has been fixed.+        if isinstance(self._fill_value, ndarray):+            return self._fill_value[()]+        return self._fill_value++    def set_fill_value(self, value=None):+        """+        Set the filling value of the masked array.++        Parameters+        ----------+        value : scalar, optional+            The new filling value. Default is None, in which case a default+            based on the data type is used.++        See Also+        --------+        ma.set_fill_value : Equivalent function.++        Examples+        --------+        >>> x = np.ma.array([0, 1.], fill_value=-np.inf)+        >>> x.fill_value+        -inf+        >>> x.set_fill_value(np.pi)+        >>> x.fill_value+        3.1415926535897931++        Reset to default:++        >>> x.set_fill_value()+        >>> x.fill_value+        1e+20++        """+        target = _check_fill_value(value, self.dtype)+        _fill_value = self._fill_value+        if _fill_value is None:+            # Create the attribute if it was undefined+            self._fill_value = target+        else:+            # Don't overwrite the attribute, just fill it (for propagation)+            _fill_value[()] = target++    fill_value = property(fget=get_fill_value, fset=set_fill_value,+                          doc="Filling value.")++    def filled(self, fill_value=None):+        """+        Return a copy of self, with masked values filled with a given value.+        **However**, if there are no masked values to fill, self will be+        returned instead as an ndarray.++        Parameters+        ----------+        fill_value : scalar, optional+            The value to use for invalid entries (None by default).+            If None, the `fill_value` attribute of the array is used instead.++        Returns+        -------+        filled_array : ndarray+            A copy of ``self`` with invalid entries replaced by *fill_value*+            (be it the function argument or the attribute of ``self``), or+            ``self`` itself as an ndarray if there are no invalid entries to+            be replaced.++        Notes+        -----+        The result is **not** a MaskedArray!++        Examples+        --------+        >>> x = np.ma.array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999)+        >>> x.filled()+        array([1, 2, -999, 4, -999])+        >>> type(x.filled())+        <type 'numpy.ndarray'>++        Subclassing is preserved. This means that if, e.g., the data part of+        the masked array is a recarray, `filled` returns a recarray:++        >>> x = np.array([(-1, 2), (-3, 4)], dtype='i8,i8').view(np.recarray)+        >>> m = np.ma.array(x, mask=[(True, False), (False, True)])+        >>> m.filled()+        rec.array([(999999,      2), (    -3, 999999)],+                  dtype=[('f0', '<i8'), ('f1', '<i8')])+        """+        m = self._mask+        if m is nomask:+            return self._data++        if fill_value is None:+            fill_value = self.fill_value+        else:+            fill_value = _check_fill_value(fill_value, self.dtype)++        if self is masked_singleton:+            return np.asanyarray(fill_value)++        if m.dtype.names is not None:+            result = self._data.copy('K')+            _recursive_filled(result, self._mask, fill_value)+        elif not m.any():+            return self._data+        else:+            result = self._data.copy('K')+            try:+                np.copyto(result, fill_value, where=m)+            except (TypeError, AttributeError):+                fill_value = narray(fill_value, dtype=object)+                d = result.astype(object)+                result = np.choose(m, (d, fill_value))+            except IndexError:+                # ok, if scalar+                if self._data.shape:+                    raise+                elif m:+                    result = np.array(fill_value, dtype=self.dtype)+                else:+                    result = self._data+        return result++    def compressed(self):+        """+        Return all the non-masked data as a 1-D array.++        Returns+        -------+        data : ndarray+            A new `ndarray` holding the non-masked data is returned.++        Notes+        -----+        The result is **not** a MaskedArray!++        Examples+        --------+        >>> x = np.ma.array(np.arange(5), mask=[0]*2 + [1]*3)+        >>> x.compressed()+        array([0, 1])+        >>> type(x.compressed())+        <type 'numpy.ndarray'>++        """+        data = ndarray.ravel(self._data)+        if self._mask is not nomask:+            data = data.compress(np.logical_not(ndarray.ravel(self._mask)))+        return data++    def compress(self, condition, axis=None, out=None):+        """+        Return `a` where condition is ``True``.++        If condition is a `MaskedArray`, missing values are considered+        as ``False``.++        Parameters+        ----------+        condition : var+            Boolean 1-d array selecting which entries to return. If len(condition)+            is less than the size of a along the axis, then output is truncated+            to length of condition array.+        axis : {None, int}, optional+            Axis along which the operation must be performed.+        out : {None, ndarray}, optional+            Alternative output array in which to place the result. It must have+            the same shape as the expected output but the type will be cast if+            necessary.++        Returns+        -------+        result : MaskedArray+            A :class:`MaskedArray` object.++        Notes+        -----+        Please note the difference with :meth:`compressed` !+        The output of :meth:`compress` has a mask, the output of+        :meth:`compressed` does not.++        Examples+        --------+        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)+        >>> print(x)+        [[1 -- 3]+         [-- 5 --]+         [7 -- 9]]+        >>> x.compress([1, 0, 1])+        masked_array(data = [1 3],+              mask = [False False],+              fill_value=999999)++        >>> x.compress([1, 0, 1], axis=1)+        masked_array(data =+         [[1 3]+         [-- --]+         [7 9]],+              mask =+         [[False False]+         [ True  True]+         [False False]],+              fill_value=999999)++        """+        # Get the basic components+        (_data, _mask) = (self._data, self._mask)++        # Force the condition to a regular ndarray and forget the missing+        # values.+        condition = np.array(condition, copy=False, subok=False)++        _new = _data.compress(condition, axis=axis, out=out).view(type(self))+        _new._update_from(self)+        if _mask is not nomask:+            _new._mask = _mask.compress(condition, axis=axis)+        return _new++    def _insert_masked_print(self):+        """+        Replace masked values with masked_print_option, casting all innermost+        dtypes to object.+        """+        if masked_print_option.enabled():+            mask = self._mask+            if mask is nomask:+                res = self._data+            else:+                # convert to object array to make filled work+                data = self._data+                # For big arrays, to avoid a costly conversion to the+                # object dtype, extract the corners before the conversion.+                print_width = (self._print_width if self.ndim > 1+                               else self._print_width_1d)+                for axis in range(self.ndim):+                    if data.shape[axis] > print_width:+                        ind = print_width // 2+                        arr = np.split(data, (ind, -ind), axis=axis)+                        data = np.concatenate((arr[0], arr[2]), axis=axis)+                        arr = np.split(mask, (ind, -ind), axis=axis)+                        mask = np.concatenate((arr[0], arr[2]), axis=axis)++                rdtype = _replace_dtype_fields(self.dtype, "O")+                res = data.astype(rdtype)+                _recursive_printoption(res, mask, masked_print_option)+        else:+            res = self.filled(self.fill_value)+        return res++    def __str__(self):+        return str(self._insert_masked_print())++    if sys.version_info.major < 3:+        def __unicode__(self):+            return unicode(self._insert_masked_print())++    def __repr__(self):+        """+        Literal string representation.++        """+        if self._baseclass is np.ndarray:+            name = 'array'+        else:+            name = self._baseclass.__name__+++        # 2016-11-19: Demoted to legacy format+        if np.get_printoptions()['legacy'] == '1.13':+            is_long = self.ndim > 1+            parameters = dict(+                name=name,+                nlen=" " * len(name),+                data=str(self),+                mask=str(self._mask),+                fill=str(self.fill_value),+                dtype=str(self.dtype)+            )+            is_structured = bool(self.dtype.names)+            key = '{}_{}'.format(+                'long' if is_long else 'short',+                'flx' if is_structured else 'std'+            )+            return _legacy_print_templates[key] % parameters++        prefix = 'masked_{}('.format(name)++        dtype_needed = (+            not np.core.arrayprint.dtype_is_implied(self.dtype) or+            np.all(self.mask) or+            self.size == 0+        )++        # determine which keyword args need to be shown+        keys = ['data', 'mask', 'fill_value']+        if dtype_needed:+            keys.append('dtype')++        # array has only one row (non-column)+        is_one_row = builtins.all(dim == 1 for dim in self.shape[:-1])++        # choose what to indent each keyword with+        min_indent = 2+        if is_one_row:+            # first key on the same line as the type, remaining keys+            # aligned by equals+            indents = {}+            indents[keys[0]] = prefix+            for k in keys[1:]:+                n = builtins.max(min_indent, len(prefix + keys[0]) - len(k))+                indents[k] = ' ' * n+            prefix = ''  # absorbed into the first indent+        else:+            # each key on its own line, indented by two spaces+            indents = {k: ' ' * min_indent for k in keys}+            prefix = prefix + '\n'  # first key on the next line++        # format the field values+        reprs = {}+        reprs['data'] = np.array2string(+            self._insert_masked_print(),+            separator=", ",+            prefix=indents['data'] + 'data=',+            suffix=',')+        reprs['mask'] = np.array2string(+            self._mask,+            separator=", ",+            prefix=indents['mask'] + 'mask=',+            suffix=',')+        reprs['fill_value'] = repr(self.fill_value)+        if dtype_needed:+            reprs['dtype'] = np.core.arrayprint.dtype_short_repr(self.dtype)++        # join keys with values and indentations+        result = ',\n'.join(+            '{}{}={}'.format(indents[k], k, reprs[k])+            for k in keys+        )+        return prefix + result + ')'++    def _delegate_binop(self, other):+        # This emulates the logic in+        #     private/binop_override.h:forward_binop_should_defer+        if isinstance(other, type(self)):+            return False+        array_ufunc = getattr(other, "__array_ufunc__", False)+        if array_ufunc is False:+            other_priority = getattr(other, "__array_priority__", -1000000)+            return self.__array_priority__ < other_priority+        else:+            # If array_ufunc is not None, it will be called inside the ufunc;+            # None explicitly tells us to not call the ufunc, i.e., defer.+            return array_ufunc is None++    def _comparison(self, other, compare):+        """Compare self with other using operator.eq or operator.ne.++        When either of the elements is masked, the result is masked as well,+        but the underlying boolean data are still set, with self and other+        considered equal if both are masked, and unequal otherwise.++        For structured arrays, all fields are combined, with masked values+        ignored. The result is masked if all fields were masked, with self+        and other considered equal only if both were fully masked.+        """+        omask = getmask(other)+        smask = self.mask+        mask = mask_or(smask, omask, copy=True)++        odata = getdata(other)+        if mask.dtype.names is not None:+            # For possibly masked structured arrays we need to be careful,+            # since the standard structured array comparison will use all+            # fields, masked or not. To avoid masked fields influencing the+            # outcome, we set all masked fields in self to other, so they'll+            # count as equal.  To prepare, we ensure we have the right shape.+            broadcast_shape = np.broadcast(self, odata).shape+            sbroadcast = np.broadcast_to(self, broadcast_shape, subok=True)+            sbroadcast._mask = mask+            sdata = sbroadcast.filled(odata)+            # Now take care of the mask; the merged mask should have an item+            # masked if all fields were masked (in one and/or other).+            mask = (mask == np.ones((), mask.dtype))++        else:+            # For regular arrays, just use the data as they come.+            sdata = self.data++        check = compare(sdata, odata)++        if isinstance(check, (np.bool_, bool)):+            return masked if mask else check++        if mask is not nomask:+            # Adjust elements that were masked, which should be treated+            # as equal if masked in both, unequal if masked in one.+            # Note that this works automatically for structured arrays too.+            check = np.where(mask, compare(smask, omask), check)+            if mask.shape != check.shape:+                # Guarantee consistency of the shape, making a copy since the+                # the mask may need to get written to later.+                mask = np.broadcast_to(mask, check.shape).copy()++        check = check.view(type(self))+        check._update_from(self)+        check._mask = mask+        return check++    def __eq__(self, other):+        """Check whether other equals self elementwise.++        When either of the elements is masked, the result is masked as well,+        but the underlying boolean data are still set, with self and other+        considered equal if both are masked, and unequal otherwise.++        For structured arrays, all fields are combined, with masked values+        ignored. The result is masked if all fields were masked, with self+        and other considered equal only if both were fully masked.+        """+        return self._comparison(other, operator.eq)++    def __ne__(self, other):+        """Check whether other does not equal self elementwise.++        When either of the elements is masked, the result is masked as well,+        but the underlying boolean data are still set, with self and other+        considered equal if both are masked, and unequal otherwise.++        For structured arrays, all fields are combined, with masked values+        ignored. The result is masked if all fields were masked, with self+        and other considered equal only if both were fully masked.+        """+        return self._comparison(other, operator.ne)++    def __add__(self, other):+        """+        Add self to other, and return a new masked array.++        """+        if self._delegate_binop(other):+            return NotImplemented+        return add(self, other)++    def __radd__(self, other):+        """+        Add other to self, and return a new masked array.++        """+        # In analogy with __rsub__ and __rdiv__, use original order:+        # we get here from `other + self`.+        return add(other, self)++    def __sub__(self, other):+        """+        Subtract other from self, and return a new masked array.++        """+        if self._delegate_binop(other):+            return NotImplemented+        return subtract(self, other)++    def __rsub__(self, other):+        """+        Subtract self from other, and return a new masked array.++        """+        return subtract(other, self)++    def __mul__(self, other):+        "Multiply self by other, and return a new masked array."+        if self._delegate_binop(other):+            return NotImplemented+        return multiply(self, other)++    def __rmul__(self, other):+        """+        Multiply other by self, and return a new masked array.++        """+        # In analogy with __rsub__ and __rdiv__, use original order:+        # we get here from `other * self`.+        return multiply(other, self)++    def __div__(self, other):+        """+        Divide other into self, and return a new masked array.++        """+        if self._delegate_binop(other):+            return NotImplemented+        return divide(self, other)++    def __truediv__(self, other):+        """+        Divide other into self, and return a new masked array.++        """+        if self._delegate_binop(other):+            return NotImplemented+        return true_divide(self, other)++    def __rtruediv__(self, other):+        """+        Divide self into other, and return a new masked array.++        """+        return true_divide(other, self)++    def __floordiv__(self, other):+        """+        Divide other into self, and return a new masked array.++        """+        if self._delegate_binop(other):+            return NotImplemented+        return floor_divide(self, other)++    def __rfloordiv__(self, other):+        """+        Divide self into other, and return a new masked array.++        """+        return floor_divide(other, self)++    def __pow__(self, other):+        """+        Raise self to the power other, masking the potential NaNs/Infs++        """+        if self._delegate_binop(other):+            return NotImplemented+        return power(self, other)++    def __rpow__(self, other):+        """+        Raise other to the power self, masking the potential NaNs/Infs++        """+        return power(other, self)++    def __iadd__(self, other):+        """+        Add other to self in-place.++        """+        m = getmask(other)+        if self._mask is nomask:+            if m is not nomask and m.any():+                self._mask = make_mask_none(self.shape, self.dtype)+                self._mask += m+        else:+            if m is not nomask:+                self._mask += m+        self._data.__iadd__(np.where(self._mask, self.dtype.type(0),+                                     getdata(other)))+        return self++    def __isub__(self, other):+        """+        Subtract other from self in-place.++        """+        m = getmask(other)+        if self._mask is nomask:+            if m is not nomask and m.any():+                self._mask = make_mask_none(self.shape, self.dtype)+                self._mask += m+        elif m is not nomask:+            self._mask += m+        self._data.__isub__(np.where(self._mask, self.dtype.type(0),+                                     getdata(other)))+        return self++    def __imul__(self, other):+        """+        Multiply self by other in-place.++        """+        m = getmask(other)+        if self._mask is nomask:+            if m is not nomask and m.any():+                self._mask = make_mask_none(self.shape, self.dtype)+                self._mask += m+        elif m is not nomask:+            self._mask += m+        self._data.__imul__(np.where(self._mask, self.dtype.type(1),+                                     getdata(other)))+        return self++    def __idiv__(self, other):+        """+        Divide self by other in-place.++        """+        other_data = getdata(other)+        dom_mask = _DomainSafeDivide().__call__(self._data, other_data)+        other_mask = getmask(other)+        new_mask = mask_or(other_mask, dom_mask)+        # The following 3 lines control the domain filling+        if dom_mask.any():+            (_, fval) = ufunc_fills[np.divide]+            other_data = np.where(dom_mask, fval, other_data)+        self._mask |= new_mask+        self._data.__idiv__(np.where(self._mask, self.dtype.type(1),+                                     other_data))+        return self++    def __ifloordiv__(self, other):+        """+        Floor divide self by other in-place.++        """+        other_data = getdata(other)+        dom_mask = _DomainSafeDivide().__call__(self._data, other_data)+        other_mask = getmask(other)+        new_mask = mask_or(other_mask, dom_mask)+        # The following 3 lines control the domain filling+        if dom_mask.any():+            (_, fval) = ufunc_fills[np.floor_divide]+            other_data = np.where(dom_mask, fval, other_data)+        self._mask |= new_mask+        self._data.__ifloordiv__(np.where(self._mask, self.dtype.type(1),+                                          other_data))+        return self++    def __itruediv__(self, other):+        """+        True divide self by other in-place.++        """+        other_data = getdata(other)+        dom_mask = _DomainSafeDivide().__call__(self._data, other_data)+        other_mask = getmask(other)+        new_mask = mask_or(other_mask, dom_mask)+        # The following 3 lines control the domain filling+        if dom_mask.any():+            (_, fval) = ufunc_fills[np.true_divide]+            other_data = np.where(dom_mask, fval, other_data)+        self._mask |= new_mask+        self._data.__itruediv__(np.where(self._mask, self.dtype.type(1),+                                         other_data))+        return self++    def __ipow__(self, other):+        """+        Raise self to the power other, in place.++        """+        other_data = getdata(other)+        other_mask = getmask(other)+        with np.errstate(divide='ignore', invalid='ignore'):+            self._data.__ipow__(np.where(self._mask, self.dtype.type(1),+                                         other_data))+        invalid = np.logical_not(np.isfinite(self._data))+        if invalid.any():+            if self._mask is not nomask:+                self._mask |= invalid+            else:+                self._mask = invalid+            np.copyto(self._data, self.fill_value, where=invalid)+        new_mask = mask_or(other_mask, invalid)+        self._mask = mask_or(self._mask, new_mask)+        return self++    def __float__(self):+        """+        Convert to float.++        """+        if self.size > 1:+            raise TypeError("Only length-1 arrays can be converted "+                            "to Python scalars")+        elif self._mask:+            warnings.warn("Warning: converting a masked element to nan.", stacklevel=2)+            return np.nan+        return float(self.item())++    def __int__(self):+        """+        Convert to int.++        """+        if self.size > 1:+            raise TypeError("Only length-1 arrays can be converted "+                            "to Python scalars")+        elif self._mask:+            raise MaskError('Cannot convert masked element to a Python int.')+        return int(self.item())++    def __long__(self):+        """+        Convert to long.+        """+        if self.size > 1:+            raise TypeError("Only length-1 arrays can be converted "+                            "to Python scalars")+        elif self._mask:+            raise MaskError('Cannot convert masked element to a Python long.')+        return long(self.item())+++    def get_imag(self):+        """+        Return the imaginary part of the masked array.++        The returned array is a view on the imaginary part of the `MaskedArray`+        whose `get_imag` method is called.++        Parameters+        ----------+        None++        Returns+        -------+        result : MaskedArray+            The imaginary part of the masked array.++        See Also+        --------+        get_real, real, imag++        Examples+        --------+        >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False])+        >>> x.get_imag()+        masked_array(data = [1.0 -- 1.6],+                     mask = [False  True False],+               fill_value = 1e+20)++        """+        result = self._data.imag.view(type(self))+        result.__setmask__(self._mask)+        return result++    imag = property(fget=get_imag, doc="Imaginary part.")++    def get_real(self):+        """+        Return the real part of the masked array.++        The returned array is a view on the real part of the `MaskedArray`+        whose `get_real` method is called.++        Parameters+        ----------+        None++        Returns+        -------+        result : MaskedArray+            The real part of the masked array.++        See Also+        --------+        get_imag, real, imag++        Examples+        --------+        >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False])+        >>> x.get_real()+        masked_array(data = [1.0 -- 3.45],+                     mask = [False  True False],+               fill_value = 1e+20)++        """+        result = self._data.real.view(type(self))+        result.__setmask__(self._mask)+        return result+    real = property(fget=get_real, doc="Real part")++    def count(self, axis=None, keepdims=np._NoValue):+        """+        Count the non-masked elements of the array along the given axis.++        Parameters+        ----------+        axis : None or int or tuple of ints, optional+            Axis or axes along which the count is performed.+            The default (`axis` = `None`) performs the count over all+            the dimensions of the input array. `axis` may be negative, in+            which case it counts from the last to the first axis.++            .. versionadded:: 1.10.0++            If this is a tuple of ints, the count is performed on multiple+            axes, instead of a single axis or all the axes as before.+        keepdims : bool, optional+            If this is set to True, the axes which are reduced are left+            in the result as dimensions with size one. With this option,+            the result will broadcast correctly against the array.++        Returns+        -------+        result : ndarray or scalar+            An array with the same shape as the input array, with the specified+            axis removed. If the array is a 0-d array, or if `axis` is None, a+            scalar is returned.++        See Also+        --------+        count_masked : Count masked elements in array or along a given axis.++        Examples+        --------+        >>> import numpy.ma as ma+        >>> a = ma.arange(6).reshape((2, 3))+        >>> a[1, :] = ma.masked+        >>> a+        masked_array(data =+         [[0 1 2]+         [-- -- --]],+                     mask =+         [[False False False]+         [ True  True  True]],+               fill_value = 999999)+        >>> a.count()+        3++        When the `axis` keyword is specified an array of appropriate size is+        returned.++        >>> a.count(axis=0)+        array([1, 1, 1])+        >>> a.count(axis=1)+        array([3, 0])++        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        m = self._mask+        # special case for matrices (we assume no other subclasses modify+        # their dimensions)+        if isinstance(self.data, np.matrix):+            if m is nomask:+                m = np.zeros(self.shape, dtype=np.bool_)+            m = m.view(type(self.data))++        if m is nomask:+            # compare to _count_reduce_items in _methods.py++            if self.shape is ():+                if axis not in (None, 0):+                    raise np.AxisError(axis=axis, ndim=self.ndim)+                return 1+            elif axis is None:+                if kwargs.get('keepdims', False):+                    return np.array(self.size, dtype=np.intp, ndmin=self.ndim)+                return self.size++            axes = normalize_axis_tuple(axis, self.ndim)+            items = 1+            for ax in axes:+                items *= self.shape[ax]++            if kwargs.get('keepdims', False):+                out_dims = list(self.shape)+                for a in axes:+                    out_dims[a] = 1+            else:+                out_dims = [d for n, d in enumerate(self.shape)+                            if n not in axes]+            # make sure to return a 0-d array if axis is supplied+            return np.full(out_dims, items, dtype=np.intp)++        # take care of the masked singleton+        if self is masked:+            return 0++        return (~m).sum(axis=axis, dtype=np.intp, **kwargs)++    def ravel(self, order='C'):+        """+        Returns a 1D version of self, as a view.++        Parameters+        ----------+        order : {'C', 'F', 'A', 'K'}, optional+            The elements of `a` are read using this index order. 'C' means to+            index the elements in C-like order, with the last axis index+            changing fastest, back to the first axis index changing slowest.+            'F' means to index the elements in Fortran-like index order, with+            the first index changing fastest, and the last index changing+            slowest. Note that the 'C' and 'F' options take no account of the+            memory layout of the underlying array, and only refer to the order+            of axis indexing.  'A' means to read the elements in Fortran-like+            index order if `m` is Fortran *contiguous* in memory, C-like order+            otherwise.  'K' means to read the elements in the order they occur+            in memory, except for reversing the data when strides are negative.+            By default, 'C' index order is used.++        Returns+        -------+        MaskedArray+            Output view is of shape ``(self.size,)`` (or+            ``(np.ma.product(self.shape),)``).++        Examples+        --------+        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)+        >>> print(x)+        [[1 -- 3]+         [-- 5 --]+         [7 -- 9]]+        >>> print(x.ravel())+        [1 -- 3 -- 5 -- 7 -- 9]++        """+        r = ndarray.ravel(self._data, order=order).view(type(self))+        r._update_from(self)+        if self._mask is not nomask:+            r._mask = ndarray.ravel(self._mask, order=order).reshape(r.shape)+        else:+            r._mask = nomask+        return r+++    def reshape(self, *s, **kwargs):+        """+        Give a new shape to the array without changing its data.++        Returns a masked array containing the same data, but with a new shape.+        The result is a view on the original array; if this is not possible, a+        ValueError is raised.++        Parameters+        ----------+        shape : int or tuple of ints+            The new shape should be compatible with the original shape. If an+            integer is supplied, then the result will be a 1-D array of that+            length.+        order : {'C', 'F'}, optional+            Determines whether the array data should be viewed as in C+            (row-major) or FORTRAN (column-major) order.++        Returns+        -------+        reshaped_array : array+            A new view on the array.++        See Also+        --------+        reshape : Equivalent function in the masked array module.+        numpy.ndarray.reshape : Equivalent method on ndarray object.+        numpy.reshape : Equivalent function in the NumPy module.++        Notes+        -----+        The reshaping operation cannot guarantee that a copy will not be made,+        to modify the shape in place, use ``a.shape = s``++        Examples+        --------+        >>> x = np.ma.array([[1,2],[3,4]], mask=[1,0,0,1])+        >>> print(x)+        [[-- 2]+         [3 --]]+        >>> x = x.reshape((4,1))+        >>> print(x)+        [[--]+         [2]+         [3]+         [--]]++        """+        kwargs.update(order=kwargs.get('order', 'C'))+        result = self._data.reshape(*s, **kwargs).view(type(self))+        result._update_from(self)+        mask = self._mask+        if mask is not nomask:+            result._mask = mask.reshape(*s, **kwargs)+        return result++    def resize(self, newshape, refcheck=True, order=False):+        """+        .. warning::++            This method does nothing, except raise a ValueError exception. A+            masked array does not own its data and therefore cannot safely be+            resized in place. Use the `numpy.ma.resize` function instead.++        This method is difficult to implement safely and may be deprecated in+        future releases of NumPy.++        """+        # Note : the 'order' keyword looks broken, let's just drop it+        errmsg = "A masked array does not own its data "\+                 "and therefore cannot be resized.\n" \+                 "Use the numpy.ma.resize function instead."+        raise ValueError(errmsg)++    def put(self, indices, values, mode='raise'):+        """+        Set storage-indexed locations to corresponding values.++        Sets self._data.flat[n] = values[n] for each n in indices.+        If `values` is shorter than `indices` then it will repeat.+        If `values` has some masked values, the initial mask is updated+        in consequence, else the corresponding values are unmasked.++        Parameters+        ----------+        indices : 1-D array_like+            Target indices, interpreted as integers.+        values : array_like+            Values to place in self._data copy at target indices.+        mode : {'raise', 'wrap', 'clip'}, optional+            Specifies how out-of-bounds indices will behave.+            'raise' : raise an error.+            'wrap' : wrap around.+            'clip' : clip to the range.++        Notes+        -----+        `values` can be a scalar or length 1 array.++        Examples+        --------+        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)+        >>> print(x)+        [[1 -- 3]+         [-- 5 --]+         [7 -- 9]]+        >>> x.put([0,4,8],[10,20,30])+        >>> print(x)+        [[10 -- 3]+         [-- 20 --]+         [7 -- 30]]++        >>> x.put(4,999)+        >>> print(x)+        [[10 -- 3]+         [-- 999 --]+         [7 -- 30]]++        """+        # Hard mask: Get rid of the values/indices that fall on masked data+        if self._hardmask and self._mask is not nomask:+            mask = self._mask[indices]+            indices = narray(indices, copy=False)+            values = narray(values, copy=False, subok=True)+            values.resize(indices.shape)+            indices = indices[~mask]+            values = values[~mask]++        self._data.put(indices, values, mode=mode)++        # short circuit if neither self nor values are masked+        if self._mask is nomask and getmask(values) is nomask:+            return++        m = getmaskarray(self)++        if getmask(values) is nomask:+            m.put(indices, False, mode=mode)+        else:+            m.put(indices, values._mask, mode=mode)+        m = make_mask(m, copy=False, shrink=True)+        self._mask = m+        return++    def ids(self):+        """+        Return the addresses of the data and mask areas.++        Parameters+        ----------+        None++        Examples+        --------+        >>> x = np.ma.array([1, 2, 3], mask=[0, 1, 1])+        >>> x.ids()+        (166670640, 166659832)++        If the array has no mask, the address of `nomask` is returned. This address+        is typically not close to the data in memory:++        >>> x = np.ma.array([1, 2, 3])+        >>> x.ids()+        (166691080, 3083169284L)++        """+        if self._mask is nomask:+            return (self.ctypes.data, id(nomask))+        return (self.ctypes.data, self._mask.ctypes.data)++    def iscontiguous(self):+        """+        Return a boolean indicating whether the data is contiguous.++        Parameters+        ----------+        None++        Examples+        --------+        >>> x = np.ma.array([1, 2, 3])+        >>> x.iscontiguous()+        True++        `iscontiguous` returns one of the flags of the masked array:++        >>> x.flags+          C_CONTIGUOUS : True+          F_CONTIGUOUS : True+          OWNDATA : False+          WRITEABLE : True+          ALIGNED : True+          WRITEBACKIFCOPY : False+          UPDATEIFCOPY : False++        """+        return self.flags['CONTIGUOUS']++    def all(self, axis=None, out=None, keepdims=np._NoValue):+        """+        Returns True if all elements evaluate to True.++        The output array is masked where all the values along the given axis+        are masked: if the output would have been a scalar and that all the+        values are masked, then the output is `masked`.++        Refer to `numpy.all` for full documentation.++        See Also+        --------+        ndarray.all : corresponding function for ndarrays+        numpy.all : equivalent function++        Examples+        --------+        >>> np.ma.array([1,2,3]).all()+        True+        >>> a = np.ma.array([1,2,3], mask=True)+        >>> (a.all() is np.ma.masked)+        True++        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        mask = _check_mask_axis(self._mask, axis, **kwargs)+        if out is None:+            d = self.filled(True).all(axis=axis, **kwargs).view(type(self))+            if d.ndim:+                d.__setmask__(mask)+            elif mask:+                return masked+            return d+        self.filled(True).all(axis=axis, out=out, **kwargs)+        if isinstance(out, MaskedArray):+            if out.ndim or mask:+                out.__setmask__(mask)+        return out++    def any(self, axis=None, out=None, keepdims=np._NoValue):+        """+        Returns True if any of the elements of `a` evaluate to True.++        Masked values are considered as False during computation.++        Refer to `numpy.any` for full documentation.++        See Also+        --------+        ndarray.any : corresponding function for ndarrays+        numpy.any : equivalent function++        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        mask = _check_mask_axis(self._mask, axis, **kwargs)+        if out is None:+            d = self.filled(False).any(axis=axis, **kwargs).view(type(self))+            if d.ndim:+                d.__setmask__(mask)+            elif mask:+                d = masked+            return d+        self.filled(False).any(axis=axis, out=out, **kwargs)+        if isinstance(out, MaskedArray):+            if out.ndim or mask:+                out.__setmask__(mask)+        return out++    def nonzero(self):+        """+        Return the indices of unmasked elements that are not zero.++        Returns a tuple of arrays, one for each dimension, containing the+        indices of the non-zero elements in that dimension. The corresponding+        non-zero values can be obtained with::++            a[a.nonzero()]++        To group the indices by element, rather than dimension, use+        instead::++            np.transpose(a.nonzero())++        The result of this is always a 2d array, with a row for each non-zero+        element.++        Parameters+        ----------+        None++        Returns+        -------+        tuple_of_arrays : tuple+            Indices of elements that are non-zero.++        See Also+        --------+        numpy.nonzero :+            Function operating on ndarrays.+        flatnonzero :+            Return indices that are non-zero in the flattened version of the input+            array.+        ndarray.nonzero :+            Equivalent ndarray method.+        count_nonzero :+            Counts the number of non-zero elements in the input array.++        Examples+        --------+        >>> import numpy.ma as ma+        >>> x = ma.array(np.eye(3))+        >>> x+        masked_array(data =+         [[ 1.  0.  0.]+         [ 0.  1.  0.]+         [ 0.  0.  1.]],+              mask =+         False,+              fill_value=1e+20)+        >>> x.nonzero()+        (array([0, 1, 2]), array([0, 1, 2]))++        Masked elements are ignored.++        >>> x[1, 1] = ma.masked+        >>> x+        masked_array(data =+         [[1.0 0.0 0.0]+         [0.0 -- 0.0]+         [0.0 0.0 1.0]],+              mask =+         [[False False False]+         [False  True False]+         [False False False]],+              fill_value=1e+20)+        >>> x.nonzero()+        (array([0, 2]), array([0, 2]))++        Indices can also be grouped by element.++        >>> np.transpose(x.nonzero())+        array([[0, 0],+               [2, 2]])++        A common use for ``nonzero`` is to find the indices of an array, where+        a condition is True.  Given an array `a`, the condition `a` > 3 is a+        boolean array and since False is interpreted as 0, ma.nonzero(a > 3)+        yields the indices of the `a` where the condition is true.++        >>> a = ma.array([[1,2,3],[4,5,6],[7,8,9]])+        >>> a > 3+        masked_array(data =+         [[False False False]+         [ True  True  True]+         [ True  True  True]],+              mask =+         False,+              fill_value=999999)+        >>> ma.nonzero(a > 3)+        (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))++        The ``nonzero`` method of the condition array can also be called.++        >>> (a > 3).nonzero()+        (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))++        """+        return narray(self.filled(0), copy=False).nonzero()++    def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None):+        """+        (this docstring should be overwritten)+        """+        #!!!: implement out + test!+        m = self._mask+        if m is nomask:+            result = super(MaskedArray, self).trace(offset=offset, axis1=axis1,+                                                    axis2=axis2, out=out)+            return result.astype(dtype)+        else:+            D = self.diagonal(offset=offset, axis1=axis1, axis2=axis2)+            return D.astype(dtype).filled(0).sum(axis=-1, out=out)+    trace.__doc__ = ndarray.trace.__doc__++    def dot(self, b, out=None, strict=False):+        """+        a.dot(b, out=None)++        Masked dot product of two arrays. Note that `out` and `strict` are+        located in different positions than in `ma.dot`. In order to+        maintain compatibility with the functional version, it is+        recommended that the optional arguments be treated as keyword only.+        At some point that may be mandatory.++        .. versionadded:: 1.10.0++        Parameters+        ----------+        b : masked_array_like+            Inputs array.+        out : masked_array, optional+            Output argument. This must have the exact kind that would be+            returned if it was not used. In particular, it must have the+            right type, must be C-contiguous, and its dtype must be the+            dtype that would be returned for `ma.dot(a,b)`. This is a+            performance feature. Therefore, if these conditions are not+            met, an exception is raised, instead of attempting to be+            flexible.+        strict : bool, optional+            Whether masked data are propagated (True) or set to 0 (False)+            for the computation. Default is False.  Propagating the mask+            means that if a masked value appears in a row or column, the+            whole row or column is considered masked.++            .. versionadded:: 1.10.2++        See Also+        --------+        numpy.ma.dot : equivalent function++        """+        return dot(self, b, out=out, strict=strict)++    def sum(self, axis=None, dtype=None, out=None, keepdims=np._NoValue):+        """+        Return the sum of the array elements over the given axis.++        Masked elements are set to 0 internally.++        Refer to `numpy.sum` for full documentation.++        See Also+        --------+        ndarray.sum : corresponding function for ndarrays+        numpy.sum : equivalent function++        Examples+        --------+        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)+        >>> print(x)+        [[1 -- 3]+         [-- 5 --]+         [7 -- 9]]+        >>> print(x.sum())+        25+        >>> print(x.sum(axis=1))+        [4 5 16]+        >>> print(x.sum(axis=0))+        [8 5 12]+        >>> print(type(x.sum(axis=0, dtype=np.int64)[0]))+        <type 'numpy.int64'>++        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        _mask = self._mask+        newmask = _check_mask_axis(_mask, axis, **kwargs)+        # No explicit output+        if out is None:+            result = self.filled(0).sum(axis, dtype=dtype, **kwargs)+            rndim = getattr(result, 'ndim', 0)+            if rndim:+                result = result.view(type(self))+                result.__setmask__(newmask)+            elif newmask:+                result = masked+            return result+        # Explicit output+        result = self.filled(0).sum(axis, dtype=dtype, out=out, **kwargs)+        if isinstance(out, MaskedArray):+            outmask = getmask(out)+            if (outmask is nomask):+                outmask = out._mask = make_mask_none(out.shape)+            outmask.flat = newmask+        return out++    def cumsum(self, axis=None, dtype=None, out=None):+        """+        Return the cumulative sum of the array elements over the given axis.++        Masked values are set to 0 internally during the computation.+        However, their position is saved, and the result will be masked at+        the same locations.++        Refer to `numpy.cumsum` for full documentation.++        Notes+        -----+        The mask is lost if `out` is not a valid :class:`MaskedArray` !++        Arithmetic is modular when using integer types, and no error is+        raised on overflow.++        See Also+        --------+        ndarray.cumsum : corresponding function for ndarrays+        numpy.cumsum : equivalent function++        Examples+        --------+        >>> marr = np.ma.array(np.arange(10), mask=[0,0,0,1,1,1,0,0,0,0])+        >>> print(marr.cumsum())+        [0 1 3 -- -- -- 9 16 24 33]++        """+        result = self.filled(0).cumsum(axis=axis, dtype=dtype, out=out)+        if out is not None:+            if isinstance(out, MaskedArray):+                out.__setmask__(self.mask)+            return out+        result = result.view(type(self))+        result.__setmask__(self._mask)+        return result++    def prod(self, axis=None, dtype=None, out=None, keepdims=np._NoValue):+        """+        Return the product of the array elements over the given axis.++        Masked elements are set to 1 internally for computation.++        Refer to `numpy.prod` for full documentation.++        Notes+        -----+        Arithmetic is modular when using integer types, and no error is raised+        on overflow.++        See Also+        --------+        ndarray.prod : corresponding function for ndarrays+        numpy.prod : equivalent function+        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        _mask = self._mask+        newmask = _check_mask_axis(_mask, axis, **kwargs)+        # No explicit output+        if out is None:+            result = self.filled(1).prod(axis, dtype=dtype, **kwargs)+            rndim = getattr(result, 'ndim', 0)+            if rndim:+                result = result.view(type(self))+                result.__setmask__(newmask)+            elif newmask:+                result = masked+            return result+        # Explicit output+        result = self.filled(1).prod(axis, dtype=dtype, out=out, **kwargs)+        if isinstance(out, MaskedArray):+            outmask = getmask(out)+            if (outmask is nomask):+                outmask = out._mask = make_mask_none(out.shape)+            outmask.flat = newmask+        return out+    product = prod++    def cumprod(self, axis=None, dtype=None, out=None):+        """+        Return the cumulative product of the array elements over the given axis.++        Masked values are set to 1 internally during the computation.+        However, their position is saved, and the result will be masked at+        the same locations.++        Refer to `numpy.cumprod` for full documentation.++        Notes+        -----+        The mask is lost if `out` is not a valid MaskedArray !++        Arithmetic is modular when using integer types, and no error is+        raised on overflow.++        See Also+        --------+        ndarray.cumprod : corresponding function for ndarrays+        numpy.cumprod : equivalent function+        """+        result = self.filled(1).cumprod(axis=axis, dtype=dtype, out=out)+        if out is not None:+            if isinstance(out, MaskedArray):+                out.__setmask__(self._mask)+            return out+        result = result.view(type(self))+        result.__setmask__(self._mask)+        return result++    def mean(self, axis=None, dtype=None, out=None, keepdims=np._NoValue):+        """+        Returns the average of the array elements along given axis.++        Masked entries are ignored, and result elements which are not+        finite will be masked.++        Refer to `numpy.mean` for full documentation.++        See Also+        --------+        ndarray.mean : corresponding function for ndarrays+        numpy.mean : Equivalent function+        numpy.ma.average: Weighted average.++        Examples+        --------+        >>> a = np.ma.array([1,2,3], mask=[False, False, True])+        >>> a+        masked_array(data = [1 2 --],+                     mask = [False False  True],+               fill_value = 999999)+        >>> a.mean()+        1.5++        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        if self._mask is nomask:+            result = super(MaskedArray, self).mean(axis=axis,+                                                   dtype=dtype, **kwargs)[()]+        else:+            dsum = self.sum(axis=axis, dtype=dtype, **kwargs)+            cnt = self.count(axis=axis, **kwargs)+            if cnt.shape == () and (cnt == 0):+                result = masked+            else:+                result = dsum * 1. / cnt+        if out is not None:+            out.flat = result+            if isinstance(out, MaskedArray):+                outmask = getmask(out)+                if (outmask is nomask):+                    outmask = out._mask = make_mask_none(out.shape)+                outmask.flat = getmask(result)+            return out+        return result++    def anom(self, axis=None, dtype=None):+        """+        Compute the anomalies (deviations from the arithmetic mean)+        along the given axis.++        Returns an array of anomalies, with the same shape as the input and+        where the arithmetic mean is computed along the given axis.++        Parameters+        ----------+        axis : int, optional+            Axis over which the anomalies are taken.+            The default is to use the mean of the flattened array as reference.+        dtype : dtype, optional+            Type to use in computing the variance. For arrays of integer type+             the default is float32; for arrays of float types it is the same as+             the array type.++        See Also+        --------+        mean : Compute the mean of the array.++        Examples+        --------+        >>> a = np.ma.array([1,2,3])+        >>> a.anom()+        masked_array(data = [-1.  0.  1.],+                     mask = False,+               fill_value = 1e+20)++        """+        m = self.mean(axis, dtype)+        if m is masked:+            return m++        if not axis:+            return (self - m)+        else:+            return (self - expand_dims(m, axis))++    def var(self, axis=None, dtype=None, out=None, ddof=0,+            keepdims=np._NoValue):+        """+        Returns the variance of the array elements along given axis.++        Masked entries are ignored, and result elements which are not+        finite will be masked.++        Refer to `numpy.var` for full documentation.++        See Also+        --------+        ndarray.var : corresponding function for ndarrays+        numpy.var : Equivalent function+        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        # Easy case: nomask, business as usual+        if self._mask is nomask:+            ret = super(MaskedArray, self).var(axis=axis, dtype=dtype, out=out,+                                               ddof=ddof, **kwargs)[()]+            if out is not None:+                if isinstance(out, MaskedArray):+                    out.__setmask__(nomask)+                return out+            return ret++        # Some data are masked, yay!+        cnt = self.count(axis=axis, **kwargs) - ddof+        danom = self - self.mean(axis, dtype, keepdims=True)+        if iscomplexobj(self):+            danom = umath.absolute(danom) ** 2+        else:+            danom *= danom+        dvar = divide(danom.sum(axis, **kwargs), cnt).view(type(self))+        # Apply the mask if it's not a scalar+        if dvar.ndim:+            dvar._mask = mask_or(self._mask.all(axis, **kwargs), (cnt <= 0))+            dvar._update_from(self)+        elif getmask(dvar):+            # Make sure that masked is returned when the scalar is masked.+            dvar = masked+            if out is not None:+                if isinstance(out, MaskedArray):+                    out.flat = 0+                    out.__setmask__(True)+                elif out.dtype.kind in 'biu':+                    errmsg = "Masked data information would be lost in one or "\+                             "more location."+                    raise MaskError(errmsg)+                else:+                    out.flat = np.nan+                return out+        # In case with have an explicit output+        if out is not None:+            # Set the data+            out.flat = dvar+            # Set the mask if needed+            if isinstance(out, MaskedArray):+                out.__setmask__(dvar.mask)+            return out+        return dvar+    var.__doc__ = np.var.__doc__++    def std(self, axis=None, dtype=None, out=None, ddof=0,+            keepdims=np._NoValue):+        """+        Returns the standard deviation of the array elements along given axis.++        Masked entries are ignored.++        Refer to `numpy.std` for full documentation.++        See Also+        --------+        ndarray.std : corresponding function for ndarrays+        numpy.std : Equivalent function+        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        dvar = self.var(axis, dtype, out, ddof, **kwargs)+        if dvar is not masked:+            if out is not None:+                np.power(out, 0.5, out=out, casting='unsafe')+                return out+            dvar = sqrt(dvar)+        return dvar++    def round(self, decimals=0, out=None):+        """+        Return each element rounded to the given number of decimals.++        Refer to `numpy.around` for full documentation.++        See Also+        --------+        ndarray.around : corresponding function for ndarrays+        numpy.around : equivalent function+        """+        result = self._data.round(decimals=decimals, out=out).view(type(self))+        if result.ndim > 0:+            result._mask = self._mask+            result._update_from(self)+        elif self._mask:+            # Return masked when the scalar is masked+            result = masked+        # No explicit output: we're done+        if out is None:+            return result+        if isinstance(out, MaskedArray):+            out.__setmask__(self._mask)+        return out++    def argsort(self, axis=np._NoValue, kind='quicksort', order=None,+                endwith=True, fill_value=None):+        """+        Return an ndarray of indices that sort the array along the+        specified axis.  Masked values are filled beforehand to+        `fill_value`.++        Parameters+        ----------+        axis : int, optional+            Axis along which to sort. If None, the default, the flattened array+            is used.++            ..  versionchanged:: 1.13.0+                Previously, the default was documented to be -1, but that was+                in error. At some future date, the default will change to -1, as+                originally intended.+                Until then, the axis should be given explicitly when+                ``arr.ndim > 1``, to avoid a FutureWarning.+        kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional+            Sorting algorithm.+        order : list, optional+            When `a` is an array with fields defined, this argument specifies+            which fields to compare first, second, etc.  Not all fields need be+            specified.+        endwith : {True, False}, optional+            Whether missing values (if any) should be treated as the largest values+            (True) or the smallest values (False)+            When the array contains unmasked values at the same extremes of the+            datatype, the ordering of these values and the masked values is+            undefined.+        fill_value : {var}, optional+            Value used internally for the masked values.+            If ``fill_value`` is not None, it supersedes ``endwith``.++        Returns+        -------+        index_array : ndarray, int+            Array of indices that sort `a` along the specified axis.+            In other words, ``a[index_array]`` yields a sorted `a`.++        See Also+        --------+        MaskedArray.sort : Describes sorting algorithms used.+        lexsort : Indirect stable sort with multiple keys.+        ndarray.sort : Inplace sort.++        Notes+        -----+        See `sort` for notes on the different sorting algorithms.++        Examples+        --------+        >>> a = np.ma.array([3,2,1], mask=[False, False, True])+        >>> a+        masked_array(data = [3 2 --],+                     mask = [False False  True],+               fill_value = 999999)+        >>> a.argsort()+        array([1, 0, 2])++        """++        # 2017-04-11, Numpy 1.13.0, gh-8701: warn on axis default+        if axis is np._NoValue:+            axis = _deprecate_argsort_axis(self)++        if fill_value is None:+            if endwith:+                # nan > inf+                if np.issubdtype(self.dtype, np.floating):+                    fill_value = np.nan+                else:+                    fill_value = minimum_fill_value(self)+            else:+                fill_value = maximum_fill_value(self)++        filled = self.filled(fill_value)+        return filled.argsort(axis=axis, kind=kind, order=order)++    def argmin(self, axis=None, fill_value=None, out=None):+        """+        Return array of indices to the minimum values along the given axis.++        Parameters+        ----------+        axis : {None, integer}+            If None, the index is into the flattened array, otherwise along+            the specified axis+        fill_value : {var}, optional+            Value used to fill in the masked values.  If None, the output of+            minimum_fill_value(self._data) is used instead.+        out : {None, array}, optional+            Array into which the result can be placed. Its type is preserved+            and it must be of the right shape to hold the output.++        Returns+        -------+        ndarray or scalar+            If multi-dimension input, returns a new ndarray of indices to the+            minimum values along the given axis.  Otherwise, returns a scalar+            of index to the minimum values along the given axis.++        Examples+        --------+        >>> x = np.ma.array(arange(4), mask=[1,1,0,0])+        >>> x.shape = (2,2)+        >>> print(x)+        [[-- --]+         [2 3]]+        >>> print(x.argmin(axis=0, fill_value=-1))+        [0 0]+        >>> print(x.argmin(axis=0, fill_value=9))+        [1 1]++        """+        if fill_value is None:+            fill_value = minimum_fill_value(self)+        d = self.filled(fill_value).view(ndarray)+        return d.argmin(axis, out=out)++    def argmax(self, axis=None, fill_value=None, out=None):+        """+        Returns array of indices of the maximum values along the given axis.+        Masked values are treated as if they had the value fill_value.++        Parameters+        ----------+        axis : {None, integer}+            If None, the index is into the flattened array, otherwise along+            the specified axis+        fill_value : {var}, optional+            Value used to fill in the masked values.  If None, the output of+            maximum_fill_value(self._data) is used instead.+        out : {None, array}, optional+            Array into which the result can be placed. Its type is preserved+            and it must be of the right shape to hold the output.++        Returns+        -------+        index_array : {integer_array}++        Examples+        --------+        >>> a = np.arange(6).reshape(2,3)+        >>> a.argmax()+        5+        >>> a.argmax(0)+        array([1, 1, 1])+        >>> a.argmax(1)+        array([2, 2])++        """+        if fill_value is None:+            fill_value = maximum_fill_value(self._data)+        d = self.filled(fill_value).view(ndarray)+        return d.argmax(axis, out=out)++    def sort(self, axis=-1, kind='quicksort', order=None,+             endwith=True, fill_value=None):+        """+        Sort the array, in-place++        Parameters+        ----------+        a : array_like+            Array to be sorted.+        axis : int, optional+            Axis along which to sort. If None, the array is flattened before+            sorting. The default is -1, which sorts along the last axis.+        kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional+            Sorting algorithm. Default is 'quicksort'.+        order : list, optional+            When `a` is a structured array, this argument specifies which fields+            to compare first, second, and so on.  This list does not need to+            include all of the fields.+        endwith : {True, False}, optional+            Whether missing values (if any) should be treated as the largest values+            (True) or the smallest values (False)+            When the array contains unmasked values at the same extremes of the+            datatype, the ordering of these values and the masked values is+            undefined.+        fill_value : {var}, optional+            Value used internally for the masked values.+            If ``fill_value`` is not None, it supersedes ``endwith``.++        Returns+        -------+        sorted_array : ndarray+            Array of the same type and shape as `a`.++        See Also+        --------+        ndarray.sort : Method to sort an array in-place.+        argsort : Indirect sort.+        lexsort : Indirect stable sort on multiple keys.+        searchsorted : Find elements in a sorted array.++        Notes+        -----+        See ``sort`` for notes on the different sorting algorithms.++        Examples+        --------+        >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0])+        >>> # Default+        >>> a.sort()+        >>> print(a)+        [1 3 5 -- --]++        >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0])+        >>> # Put missing values in the front+        >>> a.sort(endwith=False)+        >>> print(a)+        [-- -- 1 3 5]++        >>> a = ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0])+        >>> # fill_value takes over endwith+        >>> a.sort(endwith=False, fill_value=3)+        >>> print(a)+        [1 -- -- 3 5]++        """+        if self._mask is nomask:+            ndarray.sort(self, axis=axis, kind=kind, order=order)+            return++        if self is masked:+            return++        sidx = self.argsort(axis=axis, kind=kind, order=order,+                            fill_value=fill_value, endwith=endwith)++        self[...] = np.take_along_axis(self, sidx, axis=axis)++    def min(self, axis=None, out=None, fill_value=None, keepdims=np._NoValue):+        """+        Return the minimum along a given axis.++        Parameters+        ----------+        axis : {None, int}, optional+            Axis along which to operate.  By default, ``axis`` is None and the+            flattened input is used.+        out : array_like, optional+            Alternative output array in which to place the result.  Must be of+            the same shape and buffer length as the expected output.+        fill_value : {var}, optional+            Value used to fill in the masked values.+            If None, use the output of `minimum_fill_value`.++        Returns+        -------+        amin : array_like+            New array holding the result.+            If ``out`` was specified, ``out`` is returned.++        See Also+        --------+        minimum_fill_value+            Returns the minimum filling value for a given datatype.++        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        _mask = self._mask+        newmask = _check_mask_axis(_mask, axis, **kwargs)+        if fill_value is None:+            fill_value = minimum_fill_value(self)+        # No explicit output+        if out is None:+            result = self.filled(fill_value).min(+                axis=axis, out=out, **kwargs).view(type(self))+            if result.ndim:+                # Set the mask+                result.__setmask__(newmask)+                # Get rid of Infs+                if newmask.ndim:+                    np.copyto(result, result.fill_value, where=newmask)+            elif newmask:+                result = masked+            return result+        # Explicit output+        result = self.filled(fill_value).min(axis=axis, out=out, **kwargs)+        if isinstance(out, MaskedArray):+            outmask = getmask(out)+            if (outmask is nomask):+                outmask = out._mask = make_mask_none(out.shape)+            outmask.flat = newmask+        else:+            if out.dtype.kind in 'biu':+                errmsg = "Masked data information would be lost in one or more"\+                         " location."+                raise MaskError(errmsg)+            np.copyto(out, np.nan, where=newmask)+        return out++    # unique to masked arrays+    def mini(self, axis=None):+        """+        Return the array minimum along the specified axis.++        .. deprecated:: 1.13.0+           This function is identical to both:++            * ``self.min(keepdims=True, axis=axis).squeeze(axis=axis)``+            * ``np.ma.minimum.reduce(self, axis=axis)``++           Typically though, ``self.min(axis=axis)`` is sufficient.++        Parameters+        ----------+        axis : int, optional+            The axis along which to find the minima. Default is None, in which case+            the minimum value in the whole array is returned.++        Returns+        -------+        min : scalar or MaskedArray+            If `axis` is None, the result is a scalar. Otherwise, if `axis` is+            given and the array is at least 2-D, the result is a masked array with+            dimension one smaller than the array on which `mini` is called.++        Examples+        --------+        >>> x = np.ma.array(np.arange(6), mask=[0 ,1, 0, 0, 0 ,1]).reshape(3, 2)+        >>> print(x)+        [[0 --]+         [2 3]+         [4 --]]+        >>> x.mini()+        0+        >>> x.mini(axis=0)+        masked_array(data = [0 3],+                     mask = [False False],+               fill_value = 999999)+        >>> print(x.mini(axis=1))+        [0 2 4]++        There is a small difference between `mini` and `min`:++        >>> x[:,1].mini(axis=0)+        masked_array(data = --,+                     mask = True,+               fill_value = 999999)+        >>> x[:,1].min(axis=0)+        masked+        """++        # 2016-04-13, 1.13.0, gh-8764+        warnings.warn(+            "`mini` is deprecated; use the `min` method or "+            "`np.ma.minimum.reduce instead.",+            DeprecationWarning, stacklevel=2)+        return minimum.reduce(self, axis)++    def max(self, axis=None, out=None, fill_value=None, keepdims=np._NoValue):+        """+        Return the maximum along a given axis.++        Parameters+        ----------+        axis : {None, int}, optional+            Axis along which to operate.  By default, ``axis`` is None and the+            flattened input is used.+        out : array_like, optional+            Alternative output array in which to place the result.  Must+            be of the same shape and buffer length as the expected output.+        fill_value : {var}, optional+            Value used to fill in the masked values.+            If None, use the output of maximum_fill_value().++        Returns+        -------+        amax : array_like+            New array holding the result.+            If ``out`` was specified, ``out`` is returned.++        See Also+        --------+        maximum_fill_value+            Returns the maximum filling value for a given datatype.++        """+        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++        _mask = self._mask+        newmask = _check_mask_axis(_mask, axis, **kwargs)+        if fill_value is None:+            fill_value = maximum_fill_value(self)+        # No explicit output+        if out is None:+            result = self.filled(fill_value).max(+                axis=axis, out=out, **kwargs).view(type(self))+            if result.ndim:+                # Set the mask+                result.__setmask__(newmask)+                # Get rid of Infs+                if newmask.ndim:+                    np.copyto(result, result.fill_value, where=newmask)+            elif newmask:+                result = masked+            return result+        # Explicit output+        result = self.filled(fill_value).max(axis=axis, out=out, **kwargs)+        if isinstance(out, MaskedArray):+            outmask = getmask(out)+            if (outmask is nomask):+                outmask = out._mask = make_mask_none(out.shape)+            outmask.flat = newmask+        else:++            if out.dtype.kind in 'biu':+                errmsg = "Masked data information would be lost in one or more"\+                         " location."+                raise MaskError(errmsg)+            np.copyto(out, np.nan, where=newmask)+        return out++    def ptp(self, axis=None, out=None, fill_value=None, keepdims=False):+        """+        Return (maximum - minimum) along the given dimension+        (i.e. peak-to-peak value).++        Parameters+        ----------+        axis : {None, int}, optional+            Axis along which to find the peaks.  If None (default) the+            flattened array is used.+        out : {None, array_like}, optional+            Alternative output array in which to place the result. It must+            have the same shape and buffer length as the expected output+            but the type will be cast if necessary.+        fill_value : {var}, optional+            Value used to fill in the masked values.++        Returns+        -------+        ptp : ndarray.+            A new array holding the result, unless ``out`` was+            specified, in which case a reference to ``out`` is returned.++        """+        if out is None:+            result = self.max(axis=axis, fill_value=fill_value,+                              keepdims=keepdims)+            result -= self.min(axis=axis, fill_value=fill_value,+                               keepdims=keepdims)+            return result+        out.flat = self.max(axis=axis, out=out, fill_value=fill_value,+                            keepdims=keepdims)+        min_value = self.min(axis=axis, fill_value=fill_value,+                             keepdims=keepdims)+        np.subtract(out, min_value, out=out, casting='unsafe')+        return out++    def partition(self, *args, **kwargs):+        warnings.warn("Warning: 'partition' will ignore the 'mask' "+                      "of the {}.".format(self.__class__.__name__),+                      stacklevel=2)+        return super(MaskedArray, self).partition(*args, **kwargs)++    def argpartition(self, *args, **kwargs):+        warnings.warn("Warning: 'argpartition' will ignore the 'mask' "+                      "of the {}.".format(self.__class__.__name__),+                      stacklevel=2)+        return super(MaskedArray, self).argpartition(*args, **kwargs)++    def take(self, indices, axis=None, out=None, mode='raise'):+        """+        """+        (_data, _mask) = (self._data, self._mask)+        cls = type(self)+        # Make sure the indices are not masked+        maskindices = getmask(indices)+        if maskindices is not nomask:+            indices = indices.filled(0)+        # Get the data, promoting scalars to 0d arrays with [...] so that+        # .view works correctly+        if out is None:+            out = _data.take(indices, axis=axis, mode=mode)[...].view(cls)+        else:+            np.take(_data, indices, axis=axis, mode=mode, out=out)+        # Get the mask+        if isinstance(out, MaskedArray):+            if _mask is nomask:+                outmask = maskindices+            else:+                outmask = _mask.take(indices, axis=axis, mode=mode)+                outmask |= maskindices+            out.__setmask__(outmask)+        # demote 0d arrays back to scalars, for consistency with ndarray.take+        return out[()]++    # Array methods+    clip = _arraymethod('clip', onmask=False)+    copy = _arraymethod('copy')+    diagonal = _arraymethod('diagonal')+    flatten = _arraymethod('flatten')+    repeat = _arraymethod('repeat')+    squeeze = _arraymethod('squeeze')+    swapaxes = _arraymethod('swapaxes')+    T = property(fget=lambda self: self.transpose())+    transpose = _arraymethod('transpose')++    def tolist(self, fill_value=None):+        """+        Return the data portion of the masked array as a hierarchical Python list.++        Data items are converted to the nearest compatible Python type.+        Masked values are converted to `fill_value`. If `fill_value` is None,+        the corresponding entries in the output list will be ``None``.++        Parameters+        ----------+        fill_value : scalar, optional+            The value to use for invalid entries. Default is None.++        Returns+        -------+        result : list+            The Python list representation of the masked array.++        Examples+        --------+        >>> x = np.ma.array([[1,2,3], [4,5,6], [7,8,9]], mask=[0] + [1,0]*4)+        >>> x.tolist()+        [[1, None, 3], [None, 5, None], [7, None, 9]]+        >>> x.tolist(-999)+        [[1, -999, 3], [-999, 5, -999], [7, -999, 9]]++        """+        _mask = self._mask+        # No mask ? Just return .data.tolist ?+        if _mask is nomask:+            return self._data.tolist()+        # Explicit fill_value: fill the array and get the list+        if fill_value is not None:+            return self.filled(fill_value).tolist()+        # Structured array.+        names = self.dtype.names+        if names:+            result = self._data.astype([(_, object) for _ in names])+            for n in names:+                result[n][_mask[n]] = None+            return result.tolist()+        # Standard arrays.+        if _mask is nomask:+            return [None]+        # Set temps to save time when dealing w/ marrays.+        inishape = self.shape+        result = np.array(self._data.ravel(), dtype=object)+        result[_mask.ravel()] = None+        result.shape = inishape+        return result.tolist()++    def tostring(self, fill_value=None, order='C'):+        """+        This function is a compatibility alias for tobytes. Despite its name it+        returns bytes not strings.+        """++        return self.tobytes(fill_value, order='C')++    def tobytes(self, fill_value=None, order='C'):+        """+        Return the array data as a string containing the raw bytes in the array.++        The array is filled with a fill value before the string conversion.++        .. versionadded:: 1.9.0++        Parameters+        ----------+        fill_value : scalar, optional+            Value used to fill in the masked values. Default is None, in which+            case `MaskedArray.fill_value` is used.+        order : {'C','F','A'}, optional+            Order of the data item in the copy. Default is 'C'.++            - 'C'   -- C order (row major).+            - 'F'   -- Fortran order (column major).+            - 'A'   -- Any, current order of array.+            - None  -- Same as 'A'.++        See Also+        --------+        ndarray.tobytes+        tolist, tofile++        Notes+        -----+        As for `ndarray.tobytes`, information about the shape, dtype, etc.,+        but also about `fill_value`, will be lost.++        Examples+        --------+        >>> x = np.ma.array(np.array([[1, 2], [3, 4]]), mask=[[0, 1], [1, 0]])+        >>> x.tobytes()+        '\\x01\\x00\\x00\\x00?B\\x0f\\x00?B\\x0f\\x00\\x04\\x00\\x00\\x00'++        """+        return self.filled(fill_value).tobytes(order=order)++    def tofile(self, fid, sep="", format="%s"):+        """+        Save a masked array to a file in binary format.++        .. warning::+          This function is not implemented yet.++        Raises+        ------+        NotImplementedError+            When `tofile` is called.++        """+        raise NotImplementedError("MaskedArray.tofile() not implemented yet.")++    def toflex(self):+        """+        Transforms a masked array into a flexible-type array.++        The flexible type array that is returned will have two fields:++        * the ``_data`` field stores the ``_data`` part of the array.+        * the ``_mask`` field stores the ``_mask`` part of the array.++        Parameters+        ----------+        None++        Returns+        -------+        record : ndarray+            A new flexible-type `ndarray` with two fields: the first element+            containing a value, the second element containing the corresponding+            mask boolean. The returned record shape matches self.shape.++        Notes+        -----+        A side-effect of transforming a masked array into a flexible `ndarray` is+        that meta information (``fill_value``, ...) will be lost.++        Examples+        --------+        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)+        >>> print(x)+        [[1 -- 3]+         [-- 5 --]+         [7 -- 9]]+        >>> print(x.toflex())+        [[(1, False) (2, True) (3, False)]+         [(4, True) (5, False) (6, True)]+         [(7, False) (8, True) (9, False)]]++        """+        # Get the basic dtype.+        ddtype = self.dtype+        # Make sure we have a mask+        _mask = self._mask+        if _mask is None:+            _mask = make_mask_none(self.shape, ddtype)+        # And get its dtype+        mdtype = self._mask.dtype++        record = np.ndarray(shape=self.shape,+                            dtype=[('_data', ddtype), ('_mask', mdtype)])+        record['_data'] = self._data+        record['_mask'] = self._mask+        return record+    torecords = toflex++    # Pickling+    def __getstate__(self):+        """Return the internal state of the masked array, for pickling+        purposes.++        """+        cf = 'CF'[self.flags.fnc]+        data_state = super(MaskedArray, self).__reduce__()[2]+        return data_state + (getmaskarray(self).tobytes(cf), self._fill_value)++    def __setstate__(self, state):+        """Restore the internal state of the masked array, for+        pickling purposes.  ``state`` is typically the output of the+        ``__getstate__`` output, and is a 5-tuple:++        - class name+        - a tuple giving the shape of the data+        - a typecode for the data+        - a binary string for the data+        - a binary string for the mask.++        """+        (_, shp, typ, isf, raw, msk, flv) = state+        super(MaskedArray, self).__setstate__((shp, typ, isf, raw))+        self._mask.__setstate__((shp, make_mask_descr(typ), isf, msk))+        self.fill_value = flv++    def __reduce__(self):+        """Return a 3-tuple for pickling a MaskedArray.++        """+        return (_mareconstruct,+                (self.__class__, self._baseclass, (0,), 'b',),+                self.__getstate__())++    def __deepcopy__(self, memo=None):+        from copy import deepcopy+        copied = MaskedArray.__new__(type(self), self, copy=True)+        if memo is None:+            memo = {}+        memo[id(self)] = copied+        for (k, v) in self.__dict__.items():+            copied.__dict__[k] = deepcopy(v, memo)+        return copied+++def _mareconstruct(subtype, baseclass, baseshape, basetype,):+    """Internal function that builds a new MaskedArray from the+    information stored in a pickle.++    """+    _data = ndarray.__new__(baseclass, baseshape, basetype)+    _mask = ndarray.__new__(ndarray, baseshape, make_mask_descr(basetype))+    return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype,)+++class mvoid(MaskedArray):+    """+    Fake a 'void' object to use for masked array with structured dtypes.+    """++    def __new__(self, data, mask=nomask, dtype=None, fill_value=None,+                hardmask=False, copy=False, subok=True):+        _data = np.array(data, copy=copy, subok=subok, dtype=dtype)+        _data = _data.view(self)+        _data._hardmask = hardmask+        if mask is not nomask:+            if isinstance(mask, np.void):+                _data._mask = mask+            else:+                try:+                    # Mask is already a 0D array+                    _data._mask = np.void(mask)+                except TypeError:+                    # Transform the mask to a void+                    mdtype = make_mask_descr(dtype)+                    _data._mask = np.array(mask, dtype=mdtype)[()]+        if fill_value is not None:+            _data.fill_value = fill_value+        return _data++    def _get_data(self):+        # Make sure that the _data part is a np.void+        return super(mvoid, self)._data[()]++    _data = property(fget=_get_data)++    def __getitem__(self, indx):+        """+        Get the index.++        """+        m = self._mask+        if isinstance(m[indx], ndarray):+            # Can happen when indx is a multi-dimensional field:+            # A = ma.masked_array(data=[([0,1],)], mask=[([True,+            #                     False],)], dtype=[("A", ">i2", (2,))])+            # x = A[0]; y = x["A"]; then y.mask["A"].size==2+            # and we can not say masked/unmasked.+            # The result is no longer mvoid!+            # See also issue #6724.+            return masked_array(+                data=self._data[indx], mask=m[indx],+                fill_value=self._fill_value[indx],+                hard_mask=self._hardmask)+        if m is not nomask and m[indx]:+            return masked+        return self._data[indx]++    def __setitem__(self, indx, value):+        self._data[indx] = value+        if self._hardmask:+            self._mask[indx] |= getattr(value, "_mask", False)+        else:+            self._mask[indx] = getattr(value, "_mask", False)++    def __str__(self):+        m = self._mask+        if m is nomask:+            return str(self._data)++        rdtype = _replace_dtype_fields(self._data.dtype, "O")+        data_arr = super(mvoid, self)._data+        res = data_arr.astype(rdtype)+        _recursive_printoption(res, self._mask, masked_print_option)+        return str(res)++    __repr__ = __str__++    def __iter__(self):+        "Defines an iterator for mvoid"+        (_data, _mask) = (self._data, self._mask)+        if _mask is nomask:+            for d in _data:+                yield d+        else:+            for (d, m) in zip(_data, _mask):+                if m:+                    yield masked+                else:+                    yield d++    def __len__(self):+        return self._data.__len__()++    def filled(self, fill_value=None):+        """+        Return a copy with masked fields filled with a given value.++        Parameters+        ----------+        fill_value : scalar, optional+            The value to use for invalid entries (None by default).+            If None, the `fill_value` attribute is used instead.++        Returns+        -------+        filled_void+            A `np.void` object++        See Also+        --------+        MaskedArray.filled++        """+        return asarray(self).filled(fill_value)[()]++    def tolist(self):+        """+    Transforms the mvoid object into a tuple.++    Masked fields are replaced by None.++    Returns+    -------+    returned_tuple+        Tuple of fields+        """+        _mask = self._mask+        if _mask is nomask:+            return self._data.tolist()+        result = []+        for (d, m) in zip(self._data, self._mask):+            if m:+                result.append(None)+            else:+                # .item() makes sure we return a standard Python object+                result.append(d.item())+        return tuple(result)+++##############################################################################+#                                Shortcuts                                   #+##############################################################################+++def isMaskedArray(x):+    """+    Test whether input is an instance of MaskedArray.++    This function returns True if `x` is an instance of MaskedArray+    and returns False otherwise.  Any object is accepted as input.++    Parameters+    ----------+    x : object+        Object to test.++    Returns+    -------+    result : bool+        True if `x` is a MaskedArray.++    See Also+    --------+    isMA : Alias to isMaskedArray.+    isarray : Alias to isMaskedArray.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.eye(3, 3)+    >>> a+    array([[ 1.,  0.,  0.],+           [ 0.,  1.,  0.],+           [ 0.,  0.,  1.]])+    >>> m = ma.masked_values(a, 0)+    >>> m+    masked_array(data =+     [[1.0 -- --]+     [-- 1.0 --]+     [-- -- 1.0]],+          mask =+     [[False  True  True]+     [ True False  True]+     [ True  True False]],+          fill_value=0.0)+    >>> ma.isMaskedArray(a)+    False+    >>> ma.isMaskedArray(m)+    True+    >>> ma.isMaskedArray([0, 1, 2])+    False++    """+    return isinstance(x, MaskedArray)+++isarray = isMaskedArray+isMA = isMaskedArray  # backward compatibility+++class MaskedConstant(MaskedArray):+    # the lone np.ma.masked instance+    __singleton = None++    @classmethod+    def __has_singleton(cls):+        # second case ensures `cls.__singleton` is not just a view on the+        # superclass singleton+        return cls.__singleton is not None and type(cls.__singleton) is cls++    def __new__(cls):+        if not cls.__has_singleton():+            # We define the masked singleton as a float for higher precedence.+            # Note that it can be tricky sometimes w/ type comparison+            data = np.array(0.)+            mask = np.array(True)++            # prevent any modifications+            data.flags.writeable = False+            mask.flags.writeable = False++            # don't fall back on MaskedArray.__new__(MaskedConstant), since+            # that might confuse it - this way, the construction is entirely+            # within our control+            cls.__singleton = MaskedArray(data, mask=mask).view(cls)++        return cls.__singleton++    def __array_finalize__(self, obj):+        if not self.__has_singleton():+            # this handles the `.view` in __new__, which we want to copy across+            # properties normally+            return super(MaskedConstant, self).__array_finalize__(obj)+        elif self is self.__singleton:+            # not clear how this can happen, play it safe+            pass+        else:+            # everywhere else, we want to downcast to MaskedArray, to prevent a+            # duplicate maskedconstant.+            self.__class__ = MaskedArray+            MaskedArray.__array_finalize__(self, obj)++    def __array_prepare__(self, obj, context=None):+        return self.view(MaskedArray).__array_prepare__(obj, context)++    def __array_wrap__(self, obj, context=None):+        return self.view(MaskedArray).__array_wrap__(obj, context)++    def __str__(self):+        return str(masked_print_option._display)++    if sys.version_info.major < 3:+        def __unicode__(self):+            return unicode(masked_print_option._display)++    def __repr__(self):+        if self is MaskedConstant.__singleton:+            return 'masked'+        else:+            # it's a subclass, or something is wrong, make it obvious+            return object.__repr__(self)++    def __reduce__(self):+        """Override of MaskedArray's __reduce__.+        """+        return (self.__class__, ())++    # inplace operations have no effect. We have to override them to avoid+    # trying to modify the readonly data and mask arrays+    def __iop__(self, other):+        return self+    __iadd__ = \+    __isub__ = \+    __imul__ = \+    __ifloordiv__ = \+    __itruediv__ = \+    __ipow__ = \+        __iop__+    del __iop__  # don't leave this around++    def copy(self, *args, **kwargs):+        """ Copy is a no-op on the maskedconstant, as it is a scalar """+        # maskedconstant is a scalar, so copy doesn't need to copy. There's+        # precedent for this with `np.bool_` scalars.+        return self++    def __copy__(self):+        return self+		+    def __deepcopy__(self, memo):+        return self++    def __setattr__(self, attr, value):+        if not self.__has_singleton():+            # allow the singleton to be initialized+            return super(MaskedConstant, self).__setattr__(attr, value)+        elif self is self.__singleton:+            raise AttributeError(+                "attributes of {!r} are not writeable".format(self))+        else:+            # duplicate instance - we can end up here from __array_finalize__,+            # where we set the __class__ attribute+            return super(MaskedConstant, self).__setattr__(attr, value)+++masked = masked_singleton = MaskedConstant()+masked_array = MaskedArray+++def array(data, dtype=None, copy=False, order=None,+          mask=nomask, fill_value=None, keep_mask=True,+          hard_mask=False, shrink=True, subok=True, ndmin=0):+    """+    Shortcut to MaskedArray.++    The options are in a different order for convenience and backwards+    compatibility.++    """+    return MaskedArray(data, mask=mask, dtype=dtype, copy=copy,+                       subok=subok, keep_mask=keep_mask,+                       hard_mask=hard_mask, fill_value=fill_value,+                       ndmin=ndmin, shrink=shrink, order=order)+array.__doc__ = masked_array.__doc__+++def is_masked(x):+    """+    Determine whether input has masked values.++    Accepts any object as input, but always returns False unless the+    input is a MaskedArray containing masked values.++    Parameters+    ----------+    x : array_like+        Array to check for masked values.++    Returns+    -------+    result : bool+        True if `x` is a MaskedArray with masked values, False otherwise.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> x = ma.masked_equal([0, 1, 0, 2, 3], 0)+    >>> x+    masked_array(data = [-- 1 -- 2 3],+          mask = [ True False  True False False],+          fill_value=999999)+    >>> ma.is_masked(x)+    True+    >>> x = ma.masked_equal([0, 1, 0, 2, 3], 42)+    >>> x+    masked_array(data = [0 1 0 2 3],+          mask = False,+          fill_value=999999)+    >>> ma.is_masked(x)+    False++    Always returns False if `x` isn't a MaskedArray.++    >>> x = [False, True, False]+    >>> ma.is_masked(x)+    False+    >>> x = 'a string'+    >>> ma.is_masked(x)+    False++    """+    m = getmask(x)+    if m is nomask:+        return False+    elif m.any():+        return True+    return False+++##############################################################################+#                             Extrema functions                              #+##############################################################################+++class _extrema_operation(_MaskedUFunc):+    """+    Generic class for maximum/minimum functions.++    .. note::+      This is the base class for `_maximum_operation` and+      `_minimum_operation`.++    """+    def __init__(self, ufunc, compare, fill_value):+        super(_extrema_operation, self).__init__(ufunc)+        self.compare = compare+        self.fill_value_func = fill_value++    def __call__(self, a, b=None):+        "Executes the call behavior."+        if b is None:+            # 2016-04-13, 1.13.0+            warnings.warn(+                "Single-argument form of np.ma.{0} is deprecated. Use "+                "np.ma.{0}.reduce instead.".format(self.__name__),+                DeprecationWarning, stacklevel=2)+            return self.reduce(a)+        return where(self.compare(a, b), a, b)++    def reduce(self, target, axis=np._NoValue):+        "Reduce target along the given axis."+        target = narray(target, copy=False, subok=True)+        m = getmask(target)++        if axis is np._NoValue and target.ndim > 1:+            # 2017-05-06, Numpy 1.13.0: warn on axis default+            warnings.warn(+                "In the future the default for ma.{0}.reduce will be axis=0, "+                "not the current None, to match np.{0}.reduce. "+                "Explicitly pass 0 or None to silence this warning.".format(+                    self.__name__+                ),+                MaskedArrayFutureWarning, stacklevel=2)+            axis = None++        if axis is not np._NoValue:+            kwargs = dict(axis=axis)+        else:+            kwargs = dict()++        if m is nomask:+            t = self.f.reduce(target, **kwargs)+        else:+            target = target.filled(+                self.fill_value_func(target)).view(type(target))+            t = self.f.reduce(target, **kwargs)+            m = umath.logical_and.reduce(m, **kwargs)+            if hasattr(t, '_mask'):+                t._mask = m+            elif m:+                t = masked+        return t++    def outer(self, a, b):+        "Return the function applied to the outer product of a and b."+        ma = getmask(a)+        mb = getmask(b)+        if ma is nomask and mb is nomask:+            m = nomask+        else:+            ma = getmaskarray(a)+            mb = getmaskarray(b)+            m = logical_or.outer(ma, mb)+        result = self.f.outer(filled(a), filled(b))+        if not isinstance(result, MaskedArray):+            result = result.view(MaskedArray)+        result._mask = m+        return result++def min(obj, axis=None, out=None, fill_value=None, keepdims=np._NoValue):+    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++    try:+        return obj.min(axis=axis, fill_value=fill_value, out=out, **kwargs)+    except (AttributeError, TypeError):+        # If obj doesn't have a min method, or if the method doesn't accept a+        # fill_value argument+        return asanyarray(obj).min(axis=axis, fill_value=fill_value,+                                   out=out, **kwargs)+min.__doc__ = MaskedArray.min.__doc__++def max(obj, axis=None, out=None, fill_value=None, keepdims=np._NoValue):+    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}++    try:+        return obj.max(axis=axis, fill_value=fill_value, out=out, **kwargs)+    except (AttributeError, TypeError):+        # If obj doesn't have a max method, or if the method doesn't accept a+        # fill_value argument+        return asanyarray(obj).max(axis=axis, fill_value=fill_value,+                                   out=out, **kwargs)+max.__doc__ = MaskedArray.max.__doc__+++def ptp(obj, axis=None, out=None, fill_value=None, keepdims=np._NoValue):+    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}+    try:+        return obj.ptp(axis, out=out, fill_value=fill_value, **kwargs)+    except (AttributeError, TypeError):+        # If obj doesn't have a ptp method or if the method doesn't accept+        # a fill_value argument+        return asanyarray(obj).ptp(axis=axis, fill_value=fill_value,+                                   out=out, **kwargs)+ptp.__doc__ = MaskedArray.ptp.__doc__+++##############################################################################+#           Definition of functions from the corresponding methods           #+##############################################################################+++class _frommethod(object):+    """+    Define functions from existing MaskedArray methods.++    Parameters+    ----------+    methodname : str+        Name of the method to transform.++    """++    def __init__(self, methodname, reversed=False):+        self.__name__ = methodname+        self.__doc__ = self.getdoc()+        self.reversed = reversed++    def getdoc(self):+        "Return the doc of the function (from the doc of the method)."+        meth = getattr(MaskedArray, self.__name__, None) or\+            getattr(np, self.__name__, None)+        signature = self.__name__ + get_object_signature(meth)+        if meth is not None:+            doc = """    %s\n%s""" % (+                signature, getattr(meth, '__doc__', None))+            return doc++    def __call__(self, a, *args, **params):+        if self.reversed:+            args = list(args)+            a, args[0] = args[0], a++        marr = asanyarray(a)+        method_name = self.__name__+        method = getattr(type(marr), method_name, None)+        if method is None:+            # use the corresponding np function+            method = getattr(np, method_name)++        return method(marr, *args, **params)+++all = _frommethod('all')+anomalies = anom = _frommethod('anom')+any = _frommethod('any')+compress = _frommethod('compress', reversed=True)+cumprod = _frommethod('cumprod')+cumsum = _frommethod('cumsum')+copy = _frommethod('copy')+diagonal = _frommethod('diagonal')+harden_mask = _frommethod('harden_mask')+ids = _frommethod('ids')+maximum = _extrema_operation(umath.maximum, greater, maximum_fill_value)+mean = _frommethod('mean')+minimum = _extrema_operation(umath.minimum, less, minimum_fill_value)+nonzero = _frommethod('nonzero')+prod = _frommethod('prod')+product = _frommethod('prod')+ravel = _frommethod('ravel')+repeat = _frommethod('repeat')+shrink_mask = _frommethod('shrink_mask')+soften_mask = _frommethod('soften_mask')+std = _frommethod('std')+sum = _frommethod('sum')+swapaxes = _frommethod('swapaxes')+#take = _frommethod('take')+trace = _frommethod('trace')+var = _frommethod('var')++count = _frommethod('count')++def take(a, indices, axis=None, out=None, mode='raise'):+    """+    """+    a = masked_array(a)+    return a.take(indices, axis=axis, out=out, mode=mode)+++def power(a, b, third=None):+    """+    Returns element-wise base array raised to power from second array.++    This is the masked array version of `numpy.power`. For details see+    `numpy.power`.++    See Also+    --------+    numpy.power++    Notes+    -----+    The *out* argument to `numpy.power` is not supported, `third` has to be+    None.++    """+    if third is not None:+        raise MaskError("3-argument power not supported.")+    # Get the masks+    ma = getmask(a)+    mb = getmask(b)+    m = mask_or(ma, mb)+    # Get the rawdata+    fa = getdata(a)+    fb = getdata(b)+    # Get the type of the result (so that we preserve subclasses)+    if isinstance(a, MaskedArray):+        basetype = type(a)+    else:+        basetype = MaskedArray+    # Get the result and view it as a (subclass of) MaskedArray+    with np.errstate(divide='ignore', invalid='ignore'):+        result = np.where(m, fa, umath.power(fa, fb)).view(basetype)+    result._update_from(a)+    # Find where we're in trouble w/ NaNs and Infs+    invalid = np.logical_not(np.isfinite(result.view(ndarray)))+    # Add the initial mask+    if m is not nomask:+        if not (result.ndim):+            return masked+        result._mask = np.logical_or(m, invalid)+    # Fix the invalid parts+    if invalid.any():+        if not result.ndim:+            return masked+        elif result._mask is nomask:+            result._mask = invalid+        result._data[invalid] = result.fill_value+    return result++argmin = _frommethod('argmin')+argmax = _frommethod('argmax')++def argsort(a, axis=np._NoValue, kind='quicksort', order=None, endwith=True, fill_value=None):+    "Function version of the eponymous method."+    a = np.asanyarray(a)++    # 2017-04-11, Numpy 1.13.0, gh-8701: warn on axis default+    if axis is np._NoValue:+        axis = _deprecate_argsort_axis(a)++    if isinstance(a, MaskedArray):+        return a.argsort(axis=axis, kind=kind, order=order,+                         endwith=endwith, fill_value=fill_value)+    else:+        return a.argsort(axis=axis, kind=kind, order=order)+argsort.__doc__ = MaskedArray.argsort.__doc__++def sort(a, axis=-1, kind='quicksort', order=None, endwith=True, fill_value=None):+    "Function version of the eponymous method."+    a = np.array(a, copy=True, subok=True)+    if axis is None:+        a = a.flatten()+        axis = 0++    if isinstance(a, MaskedArray):+        a.sort(axis=axis, kind=kind, order=order,+               endwith=endwith, fill_value=fill_value)+    else:+        a.sort(axis=axis, kind=kind, order=order)+    return a+sort.__doc__ = MaskedArray.sort.__doc__+++def compressed(x):+    """+    Return all the non-masked data as a 1-D array.++    This function is equivalent to calling the "compressed" method of a+    `MaskedArray`, see `MaskedArray.compressed` for details.++    See Also+    --------+    MaskedArray.compressed+        Equivalent method.++    """+    return asanyarray(x).compressed()+++def concatenate(arrays, axis=0):+    """+    Concatenate a sequence of arrays along the given axis.++    Parameters+    ----------+    arrays : sequence of array_like+        The arrays must have the same shape, except in the dimension+        corresponding to `axis` (the first, by default).+    axis : int, optional+        The axis along which the arrays will be joined. Default is 0.++    Returns+    -------+    result : MaskedArray+        The concatenated array with any masked entries preserved.++    See Also+    --------+    numpy.concatenate : Equivalent function in the top-level NumPy module.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = ma.arange(3)+    >>> a[1] = ma.masked+    >>> b = ma.arange(2, 5)+    >>> a+    masked_array(data = [0 -- 2],+                 mask = [False  True False],+           fill_value = 999999)+    >>> b+    masked_array(data = [2 3 4],+                 mask = False,+           fill_value = 999999)+    >>> ma.concatenate([a, b])+    masked_array(data = [0 -- 2 2 3 4],+                 mask = [False  True False False False False],+           fill_value = 999999)++    """+    d = np.concatenate([getdata(a) for a in arrays], axis)+    rcls = get_masked_subclass(*arrays)+    data = d.view(rcls)+    # Check whether one of the arrays has a non-empty mask.+    for x in arrays:+        if getmask(x) is not nomask:+            break+    else:+        return data+    # OK, so we have to concatenate the masks+    dm = np.concatenate([getmaskarray(a) for a in arrays], axis)+    dm = dm.reshape(d.shape)++    # If we decide to keep a '_shrinkmask' option, we want to check that+    # all of them are True, and then check for dm.any()+    data._mask = _shrink_mask(dm)+    return data+++def diag(v, k=0):+    """+    Extract a diagonal or construct a diagonal array.++    This function is the equivalent of `numpy.diag` that takes masked+    values into account, see `numpy.diag` for details.++    See Also+    --------+    numpy.diag : Equivalent function for ndarrays.++    """+    output = np.diag(v, k).view(MaskedArray)+    if getmask(v) is not nomask:+        output._mask = np.diag(v._mask, k)+    return output+++def left_shift(a, n):+    """+    Shift the bits of an integer to the left.++    This is the masked array version of `numpy.left_shift`, for details+    see that function.++    See Also+    --------+    numpy.left_shift++    """+    m = getmask(a)+    if m is nomask:+        d = umath.left_shift(filled(a), n)+        return masked_array(d)+    else:+        d = umath.left_shift(filled(a, 0), n)+        return masked_array(d, mask=m)+++def right_shift(a, n):+    """+    Shift the bits of an integer to the right.++    This is the masked array version of `numpy.right_shift`, for details+    see that function.++    See Also+    --------+    numpy.right_shift++    """+    m = getmask(a)+    if m is nomask:+        d = umath.right_shift(filled(a), n)+        return masked_array(d)+    else:+        d = umath.right_shift(filled(a, 0), n)+        return masked_array(d, mask=m)+++def put(a, indices, values, mode='raise'):+    """+    Set storage-indexed locations to corresponding values.++    This function is equivalent to `MaskedArray.put`, see that method+    for details.++    See Also+    --------+    MaskedArray.put++    """+    # We can't use 'frommethod', the order of arguments is different+    try:+        return a.put(indices, values, mode=mode)+    except AttributeError:+        return narray(a, copy=False).put(indices, values, mode=mode)+++def putmask(a, mask, values):  # , mode='raise'):+    """+    Changes elements of an array based on conditional and input values.++    This is the masked array version of `numpy.putmask`, for details see+    `numpy.putmask`.++    See Also+    --------+    numpy.putmask++    Notes+    -----+    Using a masked array as `values` will **not** transform a `ndarray` into+    a `MaskedArray`.++    """+    # We can't use 'frommethod', the order of arguments is different+    if not isinstance(a, MaskedArray):+        a = a.view(MaskedArray)+    (valdata, valmask) = (getdata(values), getmask(values))+    if getmask(a) is nomask:+        if valmask is not nomask:+            a._sharedmask = True+            a._mask = make_mask_none(a.shape, a.dtype)+            np.copyto(a._mask, valmask, where=mask)+    elif a._hardmask:+        if valmask is not nomask:+            m = a._mask.copy()+            np.copyto(m, valmask, where=mask)+            a.mask |= m+    else:+        if valmask is nomask:+            valmask = getmaskarray(values)+        np.copyto(a._mask, valmask, where=mask)+    np.copyto(a._data, valdata, where=mask)+    return+++def transpose(a, axes=None):+    """+    Permute the dimensions of an array.++    This function is exactly equivalent to `numpy.transpose`.++    See Also+    --------+    numpy.transpose : Equivalent function in top-level NumPy module.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> x = ma.arange(4).reshape((2,2))+    >>> x[1, 1] = ma.masked+    >>>> x+    masked_array(data =+     [[0 1]+     [2 --]],+                 mask =+     [[False False]+     [False  True]],+           fill_value = 999999)++    >>> ma.transpose(x)+    masked_array(data =+     [[0 2]+     [1 --]],+                 mask =+     [[False False]+     [False  True]],+           fill_value = 999999)++    """+    # We can't use 'frommethod', as 'transpose' doesn't take keywords+    try:+        return a.transpose(axes)+    except AttributeError:+        return narray(a, copy=False).transpose(axes).view(MaskedArray)+++def reshape(a, new_shape, order='C'):+    """+    Returns an array containing the same data with a new shape.++    Refer to `MaskedArray.reshape` for full documentation.++    See Also+    --------+    MaskedArray.reshape : equivalent function++    """+    # We can't use 'frommethod', it whine about some parameters. Dmmit.+    try:+        return a.reshape(new_shape, order=order)+    except AttributeError:+        _tmp = narray(a, copy=False).reshape(new_shape, order=order)+        return _tmp.view(MaskedArray)+++def resize(x, new_shape):+    """+    Return a new masked array with the specified size and shape.++    This is the masked equivalent of the `numpy.resize` function. The new+    array is filled with repeated copies of `x` (in the order that the+    data are stored in memory). If `x` is masked, the new array will be+    masked, and the new mask will be a repetition of the old one.++    See Also+    --------+    numpy.resize : Equivalent function in the top level NumPy module.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = ma.array([[1, 2] ,[3, 4]])+    >>> a[0, 1] = ma.masked+    >>> a+    masked_array(data =+     [[1 --]+     [3 4]],+                 mask =+     [[False  True]+     [False False]],+           fill_value = 999999)+    >>> np.resize(a, (3, 3))+    array([[1, 2, 3],+           [4, 1, 2],+           [3, 4, 1]])+    >>> ma.resize(a, (3, 3))+    masked_array(data =+     [[1 -- 3]+     [4 1 --]+     [3 4 1]],+                 mask =+     [[False  True False]+     [False False  True]+     [False False False]],+           fill_value = 999999)++    A MaskedArray is always returned, regardless of the input type.++    >>> a = np.array([[1, 2] ,[3, 4]])+    >>> ma.resize(a, (3, 3))+    masked_array(data =+     [[1 2 3]+     [4 1 2]+     [3 4 1]],+                 mask =+     False,+           fill_value = 999999)++    """+    # We can't use _frommethods here, as N.resize is notoriously whiny.+    m = getmask(x)+    if m is not nomask:+        m = np.resize(m, new_shape)+    result = np.resize(x, new_shape).view(get_masked_subclass(x))+    if result.ndim:+        result._mask = m+    return result+++def rank(obj):+    """+    maskedarray version of the numpy function.++    .. note::+        Deprecated since 1.10.0++    """+    # 2015-04-12, 1.10.0+    warnings.warn(+        "`rank` is deprecated; use the `ndim` function instead. ",+        np.VisibleDeprecationWarning, stacklevel=2)+    return np.ndim(getdata(obj))++rank.__doc__ = np.rank.__doc__+++def ndim(obj):+    """+    maskedarray version of the numpy function.++    """+    return np.ndim(getdata(obj))++ndim.__doc__ = np.ndim.__doc__+++def shape(obj):+    "maskedarray version of the numpy function."+    return np.shape(getdata(obj))+shape.__doc__ = np.shape.__doc__+++def size(obj, axis=None):+    "maskedarray version of the numpy function."+    return np.size(getdata(obj), axis)+size.__doc__ = np.size.__doc__+++##############################################################################+#                            Extra functions                                 #+##############################################################################+++def where(condition, x=_NoValue, y=_NoValue):+    """+    Return a masked array with elements from `x` or `y`, depending on condition.++    .. note::+        When only `condition` is provided, this function is identical to+        `nonzero`. The rest of this documentation covers only the case where+        all three arguments are provided.++    Parameters+    ----------+    condition : array_like, bool+        Where True, yield `x`, otherwise yield `y`. +    x, y : array_like, optional+        Values from which to choose. `x`, `y` and `condition` need to be+        broadcastable to some shape.++    Returns+    -------+    out : MaskedArray+        An masked array with `masked` elements where the condition is masked,+        elements from `x` where `condition` is True, and elements from `y`+        elsewhere.++    See Also+    --------+    numpy.where : Equivalent function in the top-level NumPy module.+    nonzero : The function that is called when x and y are omitted++    Examples+    --------+    >>> x = np.ma.array(np.arange(9.).reshape(3, 3), mask=[[0, 1, 0],+    ...                                                    [1, 0, 1],+    ...                                                    [0, 1, 0]])+    >>> print(x)+    [[0.0 -- 2.0]+     [-- 4.0 --]+     [6.0 -- 8.0]]+    >>> print(np.ma.where(x > 5, x, -3.1416))+    [[-3.1416 -- -3.1416]+     [-- -3.1416 --]+     [6.0 -- 8.0]]++    """++    # handle the single-argument case+    missing = (x is _NoValue, y is _NoValue).count(True)+    if missing == 1:+        raise ValueError("Must provide both 'x' and 'y' or neither.")+    if missing == 2:+        return nonzero(condition)++    # we only care if the condition is true - false or masked pick y+    cf = filled(condition, False)+    xd = getdata(x)+    yd = getdata(y)++    # we need the full arrays here for correct final dimensions+    cm = getmaskarray(condition)+    xm = getmaskarray(x)+    ym = getmaskarray(y)++    # deal with the fact that masked.dtype == float64, but we don't actually+    # want to treat it as that.+    if x is masked and y is not masked:+        xd = np.zeros((), dtype=yd.dtype)+        xm = np.ones((),  dtype=ym.dtype)+    elif y is masked and x is not masked:+        yd = np.zeros((), dtype=xd.dtype)+        ym = np.ones((),  dtype=xm.dtype)++    data = np.where(cf, xd, yd)+    mask = np.where(cf, xm, ym)+    mask = np.where(cm, np.ones((), dtype=mask.dtype), mask)++    # collapse the mask, for backwards compatibility+    mask = _shrink_mask(mask)++    return masked_array(data, mask=mask)+++def choose(indices, choices, out=None, mode='raise'):+    """+    Use an index array to construct a new array from a set of choices.++    Given an array of integers and a set of n choice arrays, this method+    will create a new array that merges each of the choice arrays.  Where a+    value in `a` is i, the new array will have the value that choices[i]+    contains in the same place.++    Parameters+    ----------+    a : ndarray of ints+        This array must contain integers in ``[0, n-1]``, where n is the+        number of choices.+    choices : sequence of arrays+        Choice arrays. The index array and all of the choices should be+        broadcastable to the same shape.+    out : array, optional+        If provided, the result will be inserted into this array. It should+        be of the appropriate shape and `dtype`.+    mode : {'raise', 'wrap', 'clip'}, optional+        Specifies how out-of-bounds indices will behave.++        * 'raise' : raise an error+        * 'wrap' : wrap around+        * 'clip' : clip to the range++    Returns+    -------+    merged_array : array++    See Also+    --------+    choose : equivalent function++    Examples+    --------+    >>> choice = np.array([[1,1,1], [2,2,2], [3,3,3]])+    >>> a = np.array([2, 1, 0])+    >>> np.ma.choose(a, choice)+    masked_array(data = [3 2 1],+          mask = False,+          fill_value=999999)++    """+    def fmask(x):+        "Returns the filled array, or True if masked."+        if x is masked:+            return True+        return filled(x)++    def nmask(x):+        "Returns the mask, True if ``masked``, False if ``nomask``."+        if x is masked:+            return True+        return getmask(x)+    # Get the indices.+    c = filled(indices, 0)+    # Get the masks.+    masks = [nmask(x) for x in choices]+    data = [fmask(x) for x in choices]+    # Construct the mask+    outputmask = np.choose(c, masks, mode=mode)+    outputmask = make_mask(mask_or(outputmask, getmask(indices)),+                           copy=0, shrink=True)+    # Get the choices.+    d = np.choose(c, data, mode=mode, out=out).view(MaskedArray)+    if out is not None:+        if isinstance(out, MaskedArray):+            out.__setmask__(outputmask)+        return out+    d.__setmask__(outputmask)+    return d+++def round_(a, decimals=0, out=None):+    """+    Return a copy of a, rounded to 'decimals' places.++    When 'decimals' is negative, it specifies the number of positions+    to the left of the decimal point.  The real and imaginary parts of+    complex numbers are rounded separately. Nothing is done if the+    array is not of float type and 'decimals' is greater than or equal+    to 0.++    Parameters+    ----------+    decimals : int+        Number of decimals to round to. May be negative.+    out : array_like+        Existing array to use for output.+        If not given, returns a default copy of a.++    Notes+    -----+    If out is given and does not have a mask attribute, the mask of a+    is lost!++    """+    if out is None:+        return np.round_(a, decimals, out)+    else:+        np.round_(getdata(a), decimals, out)+        if hasattr(out, '_mask'):+            out._mask = getmask(a)+        return out+round = round_+++# Needed by dot, so move here from extras.py. It will still be exported+# from extras.py for compatibility.+def mask_rowcols(a, axis=None):+    """+    Mask rows and/or columns of a 2D array that contain masked values.++    Mask whole rows and/or columns of a 2D array that contain+    masked values.  The masking behavior is selected using the+    `axis` parameter.++      - If `axis` is None, rows *and* columns are masked.+      - If `axis` is 0, only rows are masked.+      - If `axis` is 1 or -1, only columns are masked.++    Parameters+    ----------+    a : array_like, MaskedArray+        The array to mask.  If not a MaskedArray instance (or if no array+        elements are masked).  The result is a MaskedArray with `mask` set+        to `nomask` (False). Must be a 2D array.+    axis : int, optional+        Axis along which to perform the operation. If None, applies to a+        flattened version of the array.++    Returns+    -------+    a : MaskedArray+        A modified version of the input array, masked depending on the value+        of the `axis` parameter.++    Raises+    ------+    NotImplementedError+        If input array `a` is not 2D.++    See Also+    --------+    mask_rows : Mask rows of a 2D array that contain masked values.+    mask_cols : Mask cols of a 2D array that contain masked values.+    masked_where : Mask where a condition is met.++    Notes+    -----+    The input array's mask is modified by this function.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = np.zeros((3, 3), dtype=int)+    >>> a[1, 1] = 1+    >>> a+    array([[0, 0, 0],+           [0, 1, 0],+           [0, 0, 0]])+    >>> a = ma.masked_equal(a, 1)+    >>> a+    masked_array(data =+     [[0 0 0]+     [0 -- 0]+     [0 0 0]],+          mask =+     [[False False False]+     [False  True False]+     [False False False]],+          fill_value=999999)+    >>> ma.mask_rowcols(a)+    masked_array(data =+     [[0 -- 0]+     [-- -- --]+     [0 -- 0]],+          mask =+     [[False  True False]+     [ True  True  True]+     [False  True False]],+          fill_value=999999)++    """+    a = array(a, subok=False)+    if a.ndim != 2:+        raise NotImplementedError("mask_rowcols works for 2D arrays only.")+    m = getmask(a)+    # Nothing is masked: return a+    if m is nomask or not m.any():+        return a+    maskedval = m.nonzero()+    a._mask = a._mask.copy()+    if not axis:+        a[np.unique(maskedval[0])] = masked+    if axis in [None, 1, -1]:+        a[:, np.unique(maskedval[1])] = masked+    return a+++# Include masked dot here to avoid import problems in getting it from+# extras.py. Note that it is not included in __all__, but rather exported+# from extras in order to avoid backward compatibility problems.+def dot(a, b, strict=False, out=None):+    """+    Return the dot product of two arrays.++    This function is the equivalent of `numpy.dot` that takes masked values+    into account. Note that `strict` and `out` are in different position+    than in the method version. In order to maintain compatibility with the+    corresponding method, it is recommended that the optional arguments be+    treated as keyword only.  At some point that may be mandatory.++    .. note::+      Works only with 2-D arrays at the moment.+++    Parameters+    ----------+    a, b : masked_array_like+        Inputs arrays.+    strict : bool, optional+        Whether masked data are propagated (True) or set to 0 (False) for+        the computation. Default is False.  Propagating the mask means that+        if a masked value appears in a row or column, the whole row or+        column is considered masked.+    out : masked_array, optional+        Output argument. This must have the exact kind that would be returned+        if it was not used. In particular, it must have the right type, must be+        C-contiguous, and its dtype must be the dtype that would be returned+        for `dot(a,b)`. This is a performance feature. Therefore, if these+        conditions are not met, an exception is raised, instead of attempting+        to be flexible.++        .. versionadded:: 1.10.2++    See Also+    --------+    numpy.dot : Equivalent function for ndarrays.++    Examples+    --------+    >>> a = ma.array([[1, 2, 3], [4, 5, 6]], mask=[[1, 0, 0], [0, 0, 0]])+    >>> b = ma.array([[1, 2], [3, 4], [5, 6]], mask=[[1, 0], [0, 0], [0, 0]])+    >>> np.ma.dot(a, b)+    masked_array(data =+     [[21 26]+     [45 64]],+                 mask =+     [[False False]+     [False False]],+           fill_value = 999999)+    >>> np.ma.dot(a, b, strict=True)+    masked_array(data =+     [[-- --]+     [-- 64]],+                 mask =+     [[ True  True]+     [ True False]],+           fill_value = 999999)++    """+    # !!!: Works only with 2D arrays. There should be a way to get it to run+    # with higher dimension+    if strict and (a.ndim == 2) and (b.ndim == 2):+        a = mask_rowcols(a, 0)+        b = mask_rowcols(b, 1)+    am = ~getmaskarray(a)+    bm = ~getmaskarray(b)++    if out is None:+        d = np.dot(filled(a, 0), filled(b, 0))+        m = ~np.dot(am, bm)+        if d.ndim == 0:+            d = np.asarray(d)+        r = d.view(get_masked_subclass(a, b))+        r.__setmask__(m)+        return r+    else:+        d = np.dot(filled(a, 0), filled(b, 0), out._data)+        if out.mask.shape != d.shape:+            out._mask = np.empty(d.shape, MaskType)+        np.dot(am, bm, out._mask)+        np.logical_not(out._mask, out._mask)+        return out+++def inner(a, b):+    """+    Returns the inner product of a and b for arrays of floating point types.++    Like the generic NumPy equivalent the product sum is over the last dimension+    of a and b. The first argument is not conjugated.++    """+    fa = filled(a, 0)+    fb = filled(b, 0)+    if fa.ndim == 0:+        fa.shape = (1,)+    if fb.ndim == 0:+        fb.shape = (1,)+    return np.inner(fa, fb).view(MaskedArray)+inner.__doc__ = doc_note(np.inner.__doc__,+                         "Masked values are replaced by 0.")+innerproduct = inner+++def outer(a, b):+    "maskedarray version of the numpy function."+    fa = filled(a, 0).ravel()+    fb = filled(b, 0).ravel()+    d = np.outer(fa, fb)+    ma = getmask(a)+    mb = getmask(b)+    if ma is nomask and mb is nomask:+        return masked_array(d)+    ma = getmaskarray(a)+    mb = getmaskarray(b)+    m = make_mask(1 - np.outer(1 - ma, 1 - mb), copy=0)+    return masked_array(d, mask=m)+outer.__doc__ = doc_note(np.outer.__doc__,+                         "Masked values are replaced by 0.")+outerproduct = outer+++def _convolve_or_correlate(f, a, v, mode, propagate_mask):+    """+    Helper function for ma.correlate and ma.convolve+    """+    if propagate_mask:+        # results which are contributed to by either item in any pair being invalid+        mask = (+            f(getmaskarray(a), np.ones(np.shape(v), dtype=bool), mode=mode)+          | f(np.ones(np.shape(a), dtype=bool), getmaskarray(v), mode=mode)+        )+        data = f(getdata(a), getdata(v), mode=mode)+    else:+        # results which are not contributed to by any pair of valid elements+        mask = ~f(~getmaskarray(a), ~getmaskarray(v))+        data = f(filled(a, 0), filled(v, 0), mode=mode)++    return masked_array(data, mask=mask)+++def correlate(a, v, mode='valid', propagate_mask=True):+    """+    Cross-correlation of two 1-dimensional sequences.++    Parameters+    ----------+    a, v : array_like+        Input sequences.+    mode : {'valid', 'same', 'full'}, optional+        Refer to the `np.convolve` docstring.  Note that the default+        is 'valid', unlike `convolve`, which uses 'full'.+    propagate_mask : bool+        If True, then a result element is masked if any masked element contributes towards it.+        If False, then a result element is only masked if no non-masked element+        contribute towards it++    Returns+    -------+    out : MaskedArray+        Discrete cross-correlation of `a` and `v`.++    See Also+    --------+    numpy.correlate : Equivalent function in the top-level NumPy module.+    """+    return _convolve_or_correlate(np.correlate, a, v, mode, propagate_mask)+++def convolve(a, v, mode='full', propagate_mask=True):+    """+    Returns the discrete, linear convolution of two one-dimensional sequences.++    Parameters+    ----------+    a, v : array_like+        Input sequences.+    mode : {'valid', 'same', 'full'}, optional+        Refer to the `np.convolve` docstring.+    propagate_mask : bool+        If True, then if any masked element is included in the sum for a result+        element, then the result is masked.+        If False, then the result element is only masked if no non-masked cells+        contribute towards it++    Returns+    -------+    out : MaskedArray+        Discrete, linear convolution of `a` and `v`.++    See Also+    --------+    numpy.convolve : Equivalent function in the top-level NumPy module.+    """+    return _convolve_or_correlate(np.convolve, a, v, mode, propagate_mask)+++def allequal(a, b, fill_value=True):+    """+    Return True if all entries of a and b are equal, using+    fill_value as a truth value where either or both are masked.++    Parameters+    ----------+    a, b : array_like+        Input arrays to compare.+    fill_value : bool, optional+        Whether masked values in a or b are considered equal (True) or not+        (False).++    Returns+    -------+    y : bool+        Returns True if the two arrays are equal within the given+        tolerance, False otherwise. If either array contains NaN,+        then False is returned.++    See Also+    --------+    all, any+    numpy.ma.allclose++    Examples+    --------+    >>> a = ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1])+    >>> a+    masked_array(data = [10000000000.0 1e-07 --],+          mask = [False False  True],+          fill_value=1e+20)++    >>> b = array([1e10, 1e-7, -42.0])+    >>> b+    array([  1.00000000e+10,   1.00000000e-07,  -4.20000000e+01])+    >>> ma.allequal(a, b, fill_value=False)+    False+    >>> ma.allequal(a, b)+    True++    """+    m = mask_or(getmask(a), getmask(b))+    if m is nomask:+        x = getdata(a)+        y = getdata(b)+        d = umath.equal(x, y)+        return d.all()+    elif fill_value:+        x = getdata(a)+        y = getdata(b)+        d = umath.equal(x, y)+        dm = array(d, mask=m, copy=False)+        return dm.filled(True).all(None)+    else:+        return False+++def allclose(a, b, masked_equal=True, rtol=1e-5, atol=1e-8):+    """+    Returns True if two arrays are element-wise equal within a tolerance.++    This function is equivalent to `allclose` except that masked values+    are treated as equal (default) or unequal, depending on the `masked_equal`+    argument.++    Parameters+    ----------+    a, b : array_like+        Input arrays to compare.+    masked_equal : bool, optional+        Whether masked values in `a` and `b` are considered equal (True) or not+        (False). They are considered equal by default.+    rtol : float, optional+        Relative tolerance. The relative difference is equal to ``rtol * b``.+        Default is 1e-5.+    atol : float, optional+        Absolute tolerance. The absolute difference is equal to `atol`.+        Default is 1e-8.++    Returns+    -------+    y : bool+        Returns True if the two arrays are equal within the given+        tolerance, False otherwise. If either array contains NaN, then+        False is returned.++    See Also+    --------+    all, any+    numpy.allclose : the non-masked `allclose`.++    Notes+    -----+    If the following equation is element-wise True, then `allclose` returns+    True::++      absolute(`a` - `b`) <= (`atol` + `rtol` * absolute(`b`))++    Return True if all elements of `a` and `b` are equal subject to+    given tolerances.++    Examples+    --------+    >>> a = ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1])+    >>> a+    masked_array(data = [10000000000.0 1e-07 --],+                 mask = [False False  True],+           fill_value = 1e+20)+    >>> b = ma.array([1e10, 1e-8, -42.0], mask=[0, 0, 1])+    >>> ma.allclose(a, b)+    False++    >>> a = ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1])+    >>> b = ma.array([1.00001e10, 1e-9, -42.0], mask=[0, 0, 1])+    >>> ma.allclose(a, b)+    True+    >>> ma.allclose(a, b, masked_equal=False)+    False++    Masked values are not compared directly.++    >>> a = ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1])+    >>> b = ma.array([1.00001e10, 1e-9, 42.0], mask=[0, 0, 1])+    >>> ma.allclose(a, b)+    True+    >>> ma.allclose(a, b, masked_equal=False)+    False++    """+    x = masked_array(a, copy=False)+    y = masked_array(b, copy=False)++    # make sure y is an inexact type to avoid abs(MIN_INT); will cause+    # casting of x later.+    dtype = np.result_type(y, 1.)+    if y.dtype != dtype:+        y = masked_array(y, dtype=dtype, copy=False)++    m = mask_or(getmask(x), getmask(y))+    xinf = np.isinf(masked_array(x, copy=False, mask=m)).filled(False)+    # If we have some infs, they should fall at the same place.+    if not np.all(xinf == filled(np.isinf(y), False)):+        return False+    # No infs at all+    if not np.any(xinf):+        d = filled(less_equal(absolute(x - y), atol + rtol * absolute(y)),+                   masked_equal)+        return np.all(d)++    if not np.all(filled(x[xinf] == y[xinf], masked_equal)):+        return False+    x = x[~xinf]+    y = y[~xinf]++    d = filled(less_equal(absolute(x - y), atol + rtol * absolute(y)),+               masked_equal)++    return np.all(d)+++def asarray(a, dtype=None, order=None):+    """+    Convert the input to a masked array of the given data-type.++    No copy is performed if the input is already an `ndarray`. If `a` is+    a subclass of `MaskedArray`, a base class `MaskedArray` is returned.++    Parameters+    ----------+    a : array_like+        Input data, in any form that can be converted to a masked array. This+        includes lists, lists of tuples, tuples, tuples of tuples, tuples+        of lists, ndarrays and masked arrays.+    dtype : dtype, optional+        By default, the data-type is inferred from the input data.+    order : {'C', 'F'}, optional+        Whether to use row-major ('C') or column-major ('FORTRAN') memory+        representation.  Default is 'C'.++    Returns+    -------+    out : MaskedArray+        Masked array interpretation of `a`.++    See Also+    --------+    asanyarray : Similar to `asarray`, but conserves subclasses.++    Examples+    --------+    >>> x = np.arange(10.).reshape(2, 5)+    >>> x+    array([[ 0.,  1.,  2.,  3.,  4.],+           [ 5.,  6.,  7.,  8.,  9.]])+    >>> np.ma.asarray(x)+    masked_array(data =+     [[ 0.  1.  2.  3.  4.]+     [ 5.  6.  7.  8.  9.]],+                 mask =+     False,+           fill_value = 1e+20)+    >>> type(np.ma.asarray(x))+    <class 'numpy.ma.core.MaskedArray'>++    """+    order = order or 'C'+    return masked_array(a, dtype=dtype, copy=False, keep_mask=True,+                        subok=False, order=order)+++def asanyarray(a, dtype=None):+    """+    Convert the input to a masked array, conserving subclasses.++    If `a` is a subclass of `MaskedArray`, its class is conserved.+    No copy is performed if the input is already an `ndarray`.++    Parameters+    ----------+    a : array_like+        Input data, in any form that can be converted to an array.+    dtype : dtype, optional+        By default, the data-type is inferred from the input data.+    order : {'C', 'F'}, optional+        Whether to use row-major ('C') or column-major ('FORTRAN') memory+        representation.  Default is 'C'.++    Returns+    -------+    out : MaskedArray+        MaskedArray interpretation of `a`.++    See Also+    --------+    asarray : Similar to `asanyarray`, but does not conserve subclass.++    Examples+    --------+    >>> x = np.arange(10.).reshape(2, 5)+    >>> x+    array([[ 0.,  1.,  2.,  3.,  4.],+           [ 5.,  6.,  7.,  8.,  9.]])+    >>> np.ma.asanyarray(x)+    masked_array(data =+     [[ 0.  1.  2.  3.  4.]+     [ 5.  6.  7.  8.  9.]],+                 mask =+     False,+           fill_value = 1e+20)+    >>> type(np.ma.asanyarray(x))+    <class 'numpy.ma.core.MaskedArray'>++    """+    # workaround for #8666, to preserve identity. Ideally the bottom line+    # would handle this for us.+    if isinstance(a, MaskedArray) and (dtype is None or dtype == a.dtype):+        return a+    return masked_array(a, dtype=dtype, copy=False, keep_mask=True, subok=True)+++##############################################################################+#                               Pickling                                     #+##############################################################################++def _pickle_warn(method):+    # NumPy 1.15.0, 2017-12-10+    warnings.warn(+        "np.ma.{method} is deprecated, use pickle.{method} instead"+            .format(method=method),+        DeprecationWarning,+        stacklevel=3)+++def dump(a, F):+    """+    Pickle a masked array to a file.++    This is a wrapper around ``cPickle.dump``.++    Parameters+    ----------+    a : MaskedArray+        The array to be pickled.+    F : str or file-like object+        The file to pickle `a` to. If a string, the full path to the file.++    """+    _pickle_warn('dump')+    if not hasattr(F, 'readline'):+        with open(F, 'w') as F:+            pickle.dump(a, F)+    else:+        pickle.dump(a, F)+++def dumps(a):+    """+    Return a string corresponding to the pickling of a masked array.++    This is a wrapper around ``cPickle.dumps``.++    Parameters+    ----------+    a : MaskedArray+        The array for which the string representation of the pickle is+        returned.++    """+    _pickle_warn('dumps')+    return pickle.dumps(a)+++def load(F):+    """+    Wrapper around ``cPickle.load`` which accepts either a file-like object+    or a filename.++    Parameters+    ----------+    F : str or file+        The file or file name to load.++    See Also+    --------+    dump : Pickle an array++    Notes+    -----+    This is different from `numpy.load`, which does not use cPickle but loads+    the NumPy binary .npy format.++    """+    _pickle_warn('load')+    if not hasattr(F, 'readline'):+        with open(F, 'r') as F:+            return pickle.load(F)+    else:+        return pickle.load(F)+++def loads(strg):+    """+    Load a pickle from the current string.++    The result of ``cPickle.loads(strg)`` is returned.++    Parameters+    ----------+    strg : str+        The string to load.++    See Also+    --------+    dumps : Return a string corresponding to the pickling of a masked array.++    """+    _pickle_warn('loads')+    return pickle.loads(strg)+++def fromfile(file, dtype=float, count=-1, sep=''):+    raise NotImplementedError(+        "fromfile() not yet implemented for a MaskedArray.")+++def fromflex(fxarray):+    """+    Build a masked array from a suitable flexible-type array.++    The input array has to have a data-type with ``_data`` and ``_mask``+    fields. This type of array is output by `MaskedArray.toflex`.++    Parameters+    ----------+    fxarray : ndarray+        The structured input array, containing ``_data`` and ``_mask``+        fields. If present, other fields are discarded.++    Returns+    -------+    result : MaskedArray+        The constructed masked array.++    See Also+    --------+    MaskedArray.toflex : Build a flexible-type array from a masked array.++    Examples+    --------+    >>> x = np.ma.array(np.arange(9).reshape(3, 3), mask=[0] + [1, 0] * 4)+    >>> rec = x.toflex()+    >>> rec+    array([[(0, False), (1, True), (2, False)],+           [(3, True), (4, False), (5, True)],+           [(6, False), (7, True), (8, False)]],+          dtype=[('_data', '<i4'), ('_mask', '|b1')])+    >>> x2 = np.ma.fromflex(rec)+    >>> x2+    masked_array(data =+     [[0 -- 2]+     [-- 4 --]+     [6 -- 8]],+                 mask =+     [[False  True False]+     [ True False  True]+     [False  True False]],+           fill_value = 999999)++    Extra fields can be present in the structured array but are discarded:++    >>> dt = [('_data', '<i4'), ('_mask', '|b1'), ('field3', '<f4')]+    >>> rec2 = np.zeros((2, 2), dtype=dt)+    >>> rec2+    array([[(0, False, 0.0), (0, False, 0.0)],+           [(0, False, 0.0), (0, False, 0.0)]],+          dtype=[('_data', '<i4'), ('_mask', '|b1'), ('field3', '<f4')])+    >>> y = np.ma.fromflex(rec2)+    >>> y+    masked_array(data =+     [[0 0]+     [0 0]],+                 mask =+     [[False False]+     [False False]],+           fill_value = 999999)++    """+    return masked_array(fxarray['_data'], mask=fxarray['_mask'])+++class _convert2ma(object):++    """+    Convert functions from numpy to numpy.ma.++    Parameters+    ----------+        _methodname : string+            Name of the method to transform.++    """+    __doc__ = None++    def __init__(self, funcname, params=None):+        self._func = getattr(np, funcname)+        self.__doc__ = self.getdoc()+        self._extras = params or {}++    def getdoc(self):+        "Return the doc of the function (from the doc of the method)."+        doc = getattr(self._func, '__doc__', None)+        sig = get_object_signature(self._func)+        if doc:+            # Add the signature of the function at the beginning of the doc+            if sig:+                sig = "%s%s\n" % (self._func.__name__, sig)+            doc = sig + doc+        return doc++    def __call__(self, *args, **params):+        # Find the common parameters to the call and the definition+        _extras = self._extras+        common_params = set(params).intersection(_extras)+        # Drop the common parameters from the call+        for p in common_params:+            _extras[p] = params.pop(p)+        # Get the result+        result = self._func.__call__(*args, **params).view(MaskedArray)+        if "fill_value" in common_params:+            result.fill_value = _extras.get("fill_value", None)+        if "hardmask" in common_params:+            result._hardmask = bool(_extras.get("hard_mask", False))+        return result++arange = _convert2ma('arange', params=dict(fill_value=None, hardmask=False))+clip = np.clip+diff = np.diff+empty = _convert2ma('empty', params=dict(fill_value=None, hardmask=False))+empty_like = _convert2ma('empty_like')+frombuffer = _convert2ma('frombuffer')+fromfunction = _convert2ma('fromfunction')+identity = _convert2ma(+    'identity', params=dict(fill_value=None, hardmask=False))+indices = np.indices+ones = _convert2ma('ones', params=dict(fill_value=None, hardmask=False))+ones_like = np.ones_like+squeeze = np.squeeze+zeros = _convert2ma('zeros', params=dict(fill_value=None, hardmask=False))+zeros_like = np.zeros_like+++def append(a, b, axis=None):+    """Append values to the end of an array.++    .. versionadded:: 1.9.0++    Parameters+    ----------+    a : array_like+        Values are appended to a copy of this array.+    b : array_like+        These values are appended to a copy of `a`.  It must be of the+        correct shape (the same shape as `a`, excluding `axis`).  If `axis`+        is not specified, `b` can be any shape and will be flattened+        before use.+    axis : int, optional+        The axis along which `v` are appended.  If `axis` is not given,+        both `a` and `b` are flattened before use.++    Returns+    -------+    append : MaskedArray+        A copy of `a` with `b` appended to `axis`.  Note that `append`+        does not occur in-place: a new array is allocated and filled.  If+        `axis` is None, the result is a flattened array.++    See Also+    --------+    numpy.append : Equivalent function in the top-level NumPy module.++    Examples+    --------+    >>> import numpy.ma as ma+    >>> a = ma.masked_values([1, 2, 3], 2)+    >>> b = ma.masked_values([[4, 5, 6], [7, 8, 9]], 7)+    >>> print(ma.append(a, b))+    [1 -- 3 4 5 6 -- 8 9]+    """+    return concatenate([a, b], axis)
+ test/files/numpy2.py view
@@ -0,0 +1,4623 @@+from __future__ import division, absolute_import, print_function++try:+    # Accessing collections abstact classes from collections+    # has been deprecated since Python 3.3+    import collections.abc as collections_abc+except ImportError:+    import collections as collections_abc+import re+import sys+import warnings+import operator++import numpy as np+import numpy.core.numeric as _nx+from numpy.core import linspace, atleast_1d, atleast_2d, transpose+from numpy.core.numeric import (+    ones, zeros, arange, concatenate, array, asarray, asanyarray, empty,+    empty_like, ndarray, around, floor, ceil, take, dot, where, intp,+    integer, isscalar, absolute, AxisError+    )+from numpy.core.umath import (+    pi, multiply, add, arctan2, frompyfunc, cos, less_equal, sqrt, sin,+    mod, exp, log10, not_equal, subtract+    )+from numpy.core.fromnumeric import (+    ravel, nonzero, sort, partition, mean, any, sum+    )+from numpy.core.numerictypes import typecodes, number+from numpy.core.function_base import add_newdoc+from numpy.lib.twodim_base import diag+from .utils import deprecate+from numpy.core.multiarray import (+    _insert, add_docstring, bincount, normalize_axis_index, _monotonicity,+    interp as compiled_interp, interp_complex as compiled_interp_complex+    )+from numpy.core.umath import _add_newdoc_ufunc as add_newdoc_ufunc+from numpy.compat import long+from numpy.compat.py3k import basestring++if sys.version_info[0] < 3:+    # Force range to be a generator, for np.delete's usage.+    range = xrange+    import __builtin__ as builtins+else:+    import builtins++# needed in this module for compatibility+from numpy.lib.histograms import histogram, histogramdd++__all__ = [+    'select', 'piecewise', 'trim_zeros', 'copy', 'iterable', 'percentile',+    'diff', 'gradient', 'angle', 'unwrap', 'sort_complex', 'disp', 'flip',+    'rot90', 'extract', 'place', 'vectorize', 'asarray_chkfinite', 'average',+    'bincount', 'digitize', 'cov', 'corrcoef',+    'msort', 'median', 'sinc', 'hamming', 'hanning', 'bartlett',+    'blackman', 'kaiser', 'trapz', 'i0', 'add_newdoc', 'add_docstring',+    'meshgrid', 'delete', 'insert', 'append', 'interp', 'add_newdoc_ufunc',+    'quantile'+    ]+++def rot90(m, k=1, axes=(0,1)):+    """+    Rotate an array by 90 degrees in the plane specified by axes.++    Rotation direction is from the first towards the second axis.++    Parameters+    ----------+    m : array_like+        Array of two or more dimensions.+    k : integer+        Number of times the array is rotated by 90 degrees.+    axes: (2,) array_like+        The array is rotated in the plane defined by the axes.+        Axes must be different.++        .. versionadded:: 1.12.0++    Returns+    -------+    y : ndarray+        A rotated view of `m`.++    See Also+    --------+    flip : Reverse the order of elements in an array along the given axis.+    fliplr : Flip an array horizontally.+    flipud : Flip an array vertically.++    Notes+    -----+    rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1))+    rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1))++    Examples+    --------+    >>> m = np.array([[1,2],[3,4]], int)+    >>> m+    array([[1, 2],+           [3, 4]])+    >>> np.rot90(m)+    array([[2, 4],+           [1, 3]])+    >>> np.rot90(m, 2)+    array([[4, 3],+           [2, 1]])+    >>> m = np.arange(8).reshape((2,2,2))+    >>> np.rot90(m, 1, (1,2))+    array([[[1, 3],+            [0, 2]],+           [[5, 7],+            [4, 6]]])++    """+    axes = tuple(axes)+    if len(axes) != 2:+        raise ValueError("len(axes) must be 2.")++    m = asanyarray(m)++    if axes[0] == axes[1] or absolute(axes[0] - axes[1]) == m.ndim:+        raise ValueError("Axes must be different.")++    if (axes[0] >= m.ndim or axes[0] < -m.ndim+        or axes[1] >= m.ndim or axes[1] < -m.ndim):+        raise ValueError("Axes={} out of range for array of ndim={}."+            .format(axes, m.ndim))++    k %= 4++    if k == 0:+        return m[:]+    if k == 2:+        return flip(flip(m, axes[0]), axes[1])++    axes_list = arange(0, m.ndim)+    (axes_list[axes[0]], axes_list[axes[1]]) = (axes_list[axes[1]],+                                                axes_list[axes[0]])++    if k == 1:+        return transpose(flip(m,axes[1]), axes_list)+    else:+        # k == 3+        return flip(transpose(m, axes_list), axes[1])+++def flip(m, axis=None):+    """+    Reverse the order of elements in an array along the given axis.++    The shape of the array is preserved, but the elements are reordered.++    .. versionadded:: 1.12.0++    Parameters+    ----------+    m : array_like+        Input array.+    axis : None or int or tuple of ints, optional+         Axis or axes along which to flip over. The default,+         axis=None, will flip over all of the axes of the input array.+         If axis is negative it counts from the last to the first axis.++         If axis is a tuple of ints, flipping is performed on all of the axes+         specified in the tuple.++         .. versionchanged:: 1.15.0+            None and tuples of axes are supported++    Returns+    -------+    out : array_like+        A view of `m` with the entries of axis reversed.  Since a view is+        returned, this operation is done in constant time.++    See Also+    --------+    flipud : Flip an array vertically (axis=0).+    fliplr : Flip an array horizontally (axis=1).++    Notes+    -----+    flip(m, 0) is equivalent to flipud(m).++    flip(m, 1) is equivalent to fliplr(m).++    flip(m, n) corresponds to ``m[...,::-1,...]`` with ``::-1`` at position n.++    flip(m) corresponds to ``m[::-1,::-1,...,::-1]`` with ``::-1`` at all+    positions.++    flip(m, (0, 1)) corresponds to ``m[::-1,::-1,...]`` with ``::-1`` at+    position 0 and position 1.++    Examples+    --------+    >>> A = np.arange(8).reshape((2,2,2))+    >>> A+    array([[[0, 1],+            [2, 3]],+           [[4, 5],+            [6, 7]]])+    >>> flip(A, 0)+    array([[[4, 5],+            [6, 7]],+           [[0, 1],+            [2, 3]]])+    >>> flip(A, 1)+    array([[[2, 3],+            [0, 1]],+           [[6, 7],+            [4, 5]]])+    >>> np.flip(A)+    array([[[7, 6],+            [5, 4]],+           [[3, 2],+            [1, 0]]])+    >>> np.flip(A, (0, 2))+    array([[[5, 4],+            [7, 6]],+           [[1, 0],+            [3, 2]]])+    >>> A = np.random.randn(3,4,5)+    >>> np.all(flip(A,2) == A[:,:,::-1,...])+    True+    """+    if not hasattr(m, 'ndim'):+        m = asarray(m)+    if axis is None:+        indexer = (np.s_[::-1],) * m.ndim+    else:+        axis = _nx.normalize_axis_tuple(axis, m.ndim)+        indexer = [np.s_[:]] * m.ndim+        for ax in axis:+            indexer[ax] = np.s_[::-1]+        indexer = tuple(indexer)+    return m[indexer]+++def iterable(y):+    """+    Check whether or not an object can be iterated over.++    Parameters+    ----------+    y : object+      Input object.++    Returns+    -------+    b : bool+      Return ``True`` if the object has an iterator method or is a+      sequence and ``False`` otherwise.+++    Examples+    --------+    >>> np.iterable([1, 2, 3])+    True+    >>> np.iterable(2)+    False++    """+    try:+        iter(y)+    except TypeError:+        return False+    return True+++def average(a, axis=None, weights=None, returned=False):+    """+    Compute the weighted average along the specified axis.++    Parameters+    ----------+    a : array_like+        Array containing data to be averaged. If `a` is not an array, a+        conversion is attempted.+    axis : None or int or tuple of ints, optional+        Axis or axes along which to average `a`.  The default,+        axis=None, will average over all of the elements of the input array.+        If axis is negative it counts from the last to the first axis.++        .. versionadded:: 1.7.0++        If axis is a tuple of ints, averaging is performed on all of the axes+        specified in the tuple instead of a single axis or all the axes as+        before.+    weights : array_like, optional+        An array of weights associated with the values in `a`. Each value in+        `a` contributes to the average according to its associated weight.+        The weights array can either be 1-D (in which case its length must be+        the size of `a` along the given axis) or of the same shape as `a`.+        If `weights=None`, then all data in `a` are assumed to have a+        weight equal to one.+    returned : bool, optional+        Default is `False`. If `True`, the tuple (`average`, `sum_of_weights`)+        is returned, otherwise only the average is returned.+        If `weights=None`, `sum_of_weights` is equivalent to the number of+        elements over which the average is taken.+++    Returns+    -------+    retval, [sum_of_weights] : array_type or double+        Return the average along the specified axis. When `returned` is `True`,+        return a tuple with the average as the first element and the sum+        of the weights as the second element. `sum_of_weights` is of the+        same type as `retval`. The result dtype follows a genereal pattern.+        If `weights` is None, the result dtype will be that of `a` , or ``float64``+        if `a` is integral. Otherwise, if `weights` is not None and `a` is non-+        integral, the result type will be the type of lowest precision capable of+        representing values of both `a` and `weights`. If `a` happens to be+        integral, the previous rules still applies but the result dtype will+        at least be ``float64``.++    Raises+    ------+    ZeroDivisionError+        When all weights along axis are zero. See `numpy.ma.average` for a+        version robust to this type of error.+    TypeError+        When the length of 1D `weights` is not the same as the shape of `a`+        along axis.++    See Also+    --------+    mean++    ma.average : average for masked arrays -- useful if your data contains+                 "missing" values+    numpy.result_type : Returns the type that results from applying the+                        numpy type promotion rules to the arguments.++    Examples+    --------+    >>> data = range(1,5)+    >>> data+    [1, 2, 3, 4]+    >>> np.average(data)+    2.5+    >>> np.average(range(1,11), weights=range(10,0,-1))+    4.0++    >>> data = np.arange(6).reshape((3,2))+    >>> data+    array([[0, 1],+           [2, 3],+           [4, 5]])+    >>> np.average(data, axis=1, weights=[1./4, 3./4])+    array([ 0.75,  2.75,  4.75])+    >>> np.average(data, weights=[1./4, 3./4])+    +    Traceback (most recent call last):+    ...+    TypeError: Axis must be specified when shapes of a and weights differ.+    +    >>> a = np.ones(5, dtype=np.float128)+    >>> w = np.ones(5, dtype=np.complex64)+    >>> avg = np.average(a, weights=w)+    >>> print(avg.dtype)+    complex256+    """+    a = np.asanyarray(a)++    if weights is None:+        avg = a.mean(axis)+        scl = avg.dtype.type(a.size/avg.size)+    else:+        wgt = np.asanyarray(weights)++        if issubclass(a.dtype.type, (np.integer, np.bool_)):+            result_dtype = np.result_type(a.dtype, wgt.dtype, 'f8')+        else:+            result_dtype = np.result_type(a.dtype, wgt.dtype)++        # Sanity checks+        if a.shape != wgt.shape:+            if axis is None:+                raise TypeError(+                    "Axis must be specified when shapes of a and weights "+                    "differ.")+            if wgt.ndim != 1:+                raise TypeError(+                    "1D weights expected when shapes of a and weights differ.")+            if wgt.shape[0] != a.shape[axis]:+                raise ValueError(+                    "Length of weights not compatible with specified axis.")++            # setup wgt to broadcast along axis+            wgt = np.broadcast_to(wgt, (a.ndim-1)*(1,) + wgt.shape)+            wgt = wgt.swapaxes(-1, axis)++        scl = wgt.sum(axis=axis, dtype=result_dtype)+        if np.any(scl == 0.0):+            raise ZeroDivisionError(+                "Weights sum to zero, can't be normalized")++        avg = np.multiply(a, wgt, dtype=result_dtype).sum(axis)/scl++    if returned:+        if scl.shape != avg.shape:+            scl = np.broadcast_to(scl, avg.shape).copy()+        return avg, scl+    else:+        return avg+++def asarray_chkfinite(a, dtype=None, order=None):+    """Convert the input to an array, checking for NaNs or Infs.++    Parameters+    ----------+    a : array_like+        Input data, in any form that can be converted to an array.  This+        includes lists, lists of tuples, tuples, tuples of tuples, tuples+        of lists and ndarrays.  Success requires no NaNs or Infs.+    dtype : data-type, optional+        By default, the data-type is inferred from the input data.+    order : {'C', 'F'}, optional+         Whether to use row-major (C-style) or+         column-major (Fortran-style) memory representation.+         Defaults to 'C'.++    Returns+    -------+    out : ndarray+        Array interpretation of `a`.  No copy is performed if the input+        is already an ndarray.  If `a` is a subclass of ndarray, a base+        class ndarray is returned.++    Raises+    ------+    ValueError+        Raises ValueError if `a` contains NaN (Not a Number) or Inf (Infinity).++    See Also+    --------+    asarray : Create and array.+    asanyarray : Similar function which passes through subclasses.+    ascontiguousarray : Convert input to a contiguous array.+    asfarray : Convert input to a floating point ndarray.+    asfortranarray : Convert input to an ndarray with column-major+                     memory order.+    fromiter : Create an array from an iterator.+    fromfunction : Construct an array by executing a function on grid+                   positions.++    Examples+    --------+    Convert a list into an array.  If all elements are finite+    ``asarray_chkfinite`` is identical to ``asarray``.++    >>> a = [1, 2]+    >>> np.asarray_chkfinite(a, dtype=float)+    array([1., 2.])++    Raises ValueError if array_like contains Nans or Infs.++    >>> a = [1, 2, np.inf]+    >>> try:+    ...     np.asarray_chkfinite(a)+    ... except ValueError:+    ...     print('ValueError')+    ...+    ValueError++    """+    a = asarray(a, dtype=dtype, order=order)+    if a.dtype.char in typecodes['AllFloat'] and not np.isfinite(a).all():+        raise ValueError(+            "array must not contain infs or NaNs")+    return a+++def piecewise(x, condlist, funclist, *args, **kw):+    """+    Evaluate a piecewise-defined function.++    Given a set of conditions and corresponding functions, evaluate each+    function on the input data wherever its condition is true.++    Parameters+    ----------+    x : ndarray or scalar+        The input domain.+    condlist : list of bool arrays or bool scalars+        Each boolean array corresponds to a function in `funclist`.  Wherever+        `condlist[i]` is True, `funclist[i](x)` is used as the output value.++        Each boolean array in `condlist` selects a piece of `x`,+        and should therefore be of the same shape as `x`.++        The length of `condlist` must correspond to that of `funclist`.+        If one extra function is given, i.e. if+        ``len(funclist) == len(condlist) + 1``, then that extra function+        is the default value, used wherever all conditions are false.+    funclist : list of callables, f(x,*args,**kw), or scalars+        Each function is evaluated over `x` wherever its corresponding+        condition is True.  It should take a 1d array as input and give an 1d+        array or a scalar value as output.  If, instead of a callable,+        a scalar is provided then a constant function (``lambda x: scalar``) is+        assumed.+    args : tuple, optional+        Any further arguments given to `piecewise` are passed to the functions+        upon execution, i.e., if called ``piecewise(..., ..., 1, 'a')``, then+        each function is called as ``f(x, 1, 'a')``.+    kw : dict, optional+        Keyword arguments used in calling `piecewise` are passed to the+        functions upon execution, i.e., if called+        ``piecewise(..., ..., alpha=1)``, then each function is called as+        ``f(x, alpha=1)``.++    Returns+    -------+    out : ndarray+        The output is the same shape and type as x and is found by+        calling the functions in `funclist` on the appropriate portions of `x`,+        as defined by the boolean arrays in `condlist`.  Portions not covered+        by any condition have a default value of 0.+++    See Also+    --------+    choose, select, where++    Notes+    -----+    This is similar to choose or select, except that functions are+    evaluated on elements of `x` that satisfy the corresponding condition from+    `condlist`.++    The result is::++            |--+            |funclist[0](x[condlist[0]])+      out = |funclist[1](x[condlist[1]])+            |...+            |funclist[n2](x[condlist[n2]])+            |--++    Examples+    --------+    Define the sigma function, which is -1 for ``x < 0`` and +1 for ``x >= 0``.++    >>> x = np.linspace(-2.5, 2.5, 6)+    >>> np.piecewise(x, [x < 0, x >= 0], [-1, 1])+    array([-1., -1., -1.,  1.,  1.,  1.])++    Define the absolute value, which is ``-x`` for ``x <0`` and ``x`` for+    ``x >= 0``.++    >>> np.piecewise(x, [x < 0, x >= 0], [lambda x: -x, lambda x: x])+    array([ 2.5,  1.5,  0.5,  0.5,  1.5,  2.5])++    Apply the same function to a scalar value.++    >>> y = -2+    >>> np.piecewise(y, [y < 0, y >= 0], [lambda x: -x, lambda x: x])+    array(2)++    """+    x = asanyarray(x)+    n2 = len(funclist)++    # undocumented: single condition is promoted to a list of one condition+    if isscalar(condlist) or (+            not isinstance(condlist[0], (list, ndarray)) and x.ndim != 0):+        condlist = [condlist]++    condlist = array(condlist, dtype=bool)+    n = len(condlist)++    if n == n2 - 1:  # compute the "otherwise" condition.+        condelse = ~np.any(condlist, axis=0, keepdims=True)+        condlist = np.concatenate([condlist, condelse], axis=0)+        n += 1+    elif n != n2:+        raise ValueError(+            "with {} condition(s), either {} or {} functions are expected"+            .format(n, n, n+1)+        )++    y = zeros(x.shape, x.dtype)+    for k in range(n):+        item = funclist[k]+        if not isinstance(item, collections_abc.Callable):+            y[condlist[k]] = item+        else:+            vals = x[condlist[k]]+            if vals.size > 0:+                y[condlist[k]] = item(vals, *args, **kw)++    return y+++def select(condlist, choicelist, default=0):+    """+    Return an array drawn from elements in choicelist, depending on conditions.++    Parameters+    ----------+    condlist : list of bool ndarrays+        The list of conditions which determine from which array in `choicelist`+        the output elements are taken. When multiple conditions are satisfied,+        the first one encountered in `condlist` is used.+    choicelist : list of ndarrays+        The list of arrays from which the output elements are taken. It has+        to be of the same length as `condlist`.+    default : scalar, optional+        The element inserted in `output` when all conditions evaluate to False.++    Returns+    -------+    output : ndarray+        The output at position m is the m-th element of the array in+        `choicelist` where the m-th element of the corresponding array in+        `condlist` is True.++    See Also+    --------+    where : Return elements from one of two arrays depending on condition.+    take, choose, compress, diag, diagonal++    Examples+    --------+    >>> x = np.arange(10)+    >>> condlist = [x<3, x>5]+    >>> choicelist = [x, x**2]+    >>> np.select(condlist, choicelist)+    array([ 0,  1,  2,  0,  0,  0, 36, 49, 64, 81])++    """+    # Check the size of condlist and choicelist are the same, or abort.+    if len(condlist) != len(choicelist):+        raise ValueError(+            'list of cases must be same length as list of conditions')++    # Now that the dtype is known, handle the deprecated select([], []) case+    if len(condlist) == 0:+        # 2014-02-24, 1.9+        warnings.warn("select with an empty condition list is not possible"+                      "and will be deprecated",+                      DeprecationWarning, stacklevel=2)+        return np.asarray(default)[()]++    choicelist = [np.asarray(choice) for choice in choicelist]+    choicelist.append(np.asarray(default))++    # need to get the result type before broadcasting for correct scalar+    # behaviour+    dtype = np.result_type(*choicelist)++    # Convert conditions to arrays and broadcast conditions and choices+    # as the shape is needed for the result. Doing it separately optimizes+    # for example when all choices are scalars.+    condlist = np.broadcast_arrays(*condlist)+    choicelist = np.broadcast_arrays(*choicelist)++    # If cond array is not an ndarray in boolean format or scalar bool, abort.+    deprecated_ints = False+    for i in range(len(condlist)):+        cond = condlist[i]+        if cond.dtype.type is not np.bool_:+            if np.issubdtype(cond.dtype, np.integer):+                # A previous implementation accepted int ndarrays accidentally.+                # Supported here deliberately, but deprecated.+                condlist[i] = condlist[i].astype(bool)+                deprecated_ints = True+            else:+                raise ValueError(+                    'invalid entry {} in condlist: should be boolean ndarray'.format(i))++    if deprecated_ints:+        # 2014-02-24, 1.9+        msg = "select condlists containing integer ndarrays is deprecated " \+            "and will be removed in the future. Use `.astype(bool)` to " \+            "convert to bools."+        warnings.warn(msg, DeprecationWarning, stacklevel=2)++    if choicelist[0].ndim == 0:+        # This may be common, so avoid the call.+        result_shape = condlist[0].shape+    else:+        result_shape = np.broadcast_arrays(condlist[0], choicelist[0])[0].shape++    result = np.full(result_shape, choicelist[-1], dtype)++    # Use np.copyto to burn each choicelist array onto result, using the+    # corresponding condlist as a boolean mask. This is done in reverse+    # order since the first choice should take precedence.+    choicelist = choicelist[-2::-1]+    condlist = condlist[::-1]+    for choice, cond in zip(choicelist, condlist):+        np.copyto(result, choice, where=cond)++    return result+++def copy(a, order='K'):+    """+    Return an array copy of the given object.++    Parameters+    ----------+    a : array_like+        Input data.+    order : {'C', 'F', 'A', 'K'}, optional+        Controls the memory layout of the copy. 'C' means C-order,+        'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous,+        'C' otherwise. 'K' means match the layout of `a` as closely+        as possible. (Note that this function and :meth:`ndarray.copy` are very+        similar, but have different default values for their order=+        arguments.)++    Returns+    -------+    arr : ndarray+        Array interpretation of `a`.++    Notes+    -----+    This is equivalent to:++    >>> np.array(a, copy=True)  #doctest: +SKIP++    Examples+    --------+    Create an array x, with a reference y and a copy z:++    >>> x = np.array([1, 2, 3])+    >>> y = x+    >>> z = np.copy(x)++    Note that, when we modify x, y changes, but not z:++    >>> x[0] = 10+    >>> x[0] == y[0]+    True+    >>> x[0] == z[0]+    False++    """+    return array(a, order=order, copy=True)++# Basic operations+++def gradient(f, *varargs, **kwargs):+    """+    Return the gradient of an N-dimensional array.++    The gradient is computed using second order accurate central differences+    in the interior points and either first or second order accurate one-sides+    (forward or backwards) differences at the boundaries.+    The returned gradient hence has the same shape as the input array.++    Parameters+    ----------+    f : array_like+        An N-dimensional array containing samples of a scalar function.+    varargs : list of scalar or array, optional+        Spacing between f values. Default unitary spacing for all dimensions.+        Spacing can be specified using:++        1. single scalar to specify a sample distance for all dimensions.+        2. N scalars to specify a constant sample distance for each dimension.+           i.e. `dx`, `dy`, `dz`, ...+        3. N arrays to specify the coordinates of the values along each+           dimension of F. The length of the array must match the size of+           the corresponding dimension+        4. Any combination of N scalars/arrays with the meaning of 2. and 3.++        If `axis` is given, the number of varargs must equal the number of axes.+        Default: 1.++    edge_order : {1, 2}, optional+        Gradient is calculated using N-th order accurate differences+        at the boundaries. Default: 1.++        .. versionadded:: 1.9.1++    axis : None or int or tuple of ints, optional+        Gradient is calculated only along the given axis or axes+        The default (axis = None) is to calculate the gradient for all the axes+        of the input array. axis may be negative, in which case it counts from+        the last to the first axis.++        .. versionadded:: 1.11.0++    Returns+    -------+    gradient : ndarray or list of ndarray+        A set of ndarrays (or a single ndarray if there is only one dimension)+        corresponding to the derivatives of f with respect to each dimension.+        Each derivative has the same shape as f.++    Examples+    --------+    >>> f = np.array([1, 2, 4, 7, 11, 16], dtype=float)+    >>> np.gradient(f)+    array([ 1. ,  1.5,  2.5,  3.5,  4.5,  5. ])+    >>> np.gradient(f, 2)+    array([ 0.5 ,  0.75,  1.25,  1.75,  2.25,  2.5 ])++    Spacing can be also specified with an array that represents the coordinates+    of the values F along the dimensions.+    For instance a uniform spacing:++    >>> x = np.arange(f.size)+    >>> np.gradient(f, x)+    array([ 1. ,  1.5,  2.5,  3.5,  4.5,  5. ])++    Or a non uniform one:++    >>> x = np.array([0., 1., 1.5, 3.5, 4., 6.], dtype=float)+    >>> np.gradient(f, x)+    array([ 1. ,  3. ,  3.5,  6.7,  6.9,  2.5])++    For two dimensional arrays, the return will be two arrays ordered by+    axis. In this example the first array stands for the gradient in+    rows and the second one in columns direction:++    >>> np.gradient(np.array([[1, 2, 6], [3, 4, 5]], dtype=float))+    [array([[ 2.,  2., -1.],+            [ 2.,  2., -1.]]), array([[ 1. ,  2.5,  4. ],+            [ 1. ,  1. ,  1. ]])]++    In this example the spacing is also specified:+    uniform for axis=0 and non uniform for axis=1++    >>> dx = 2.+    >>> y = [1., 1.5, 3.5]+    >>> np.gradient(np.array([[1, 2, 6], [3, 4, 5]], dtype=float), dx, y)+    [array([[ 1. ,  1. , -0.5],+            [ 1. ,  1. , -0.5]]), array([[ 2. ,  2. ,  2. ],+            [ 2. ,  1.7,  0.5]])]++    It is possible to specify how boundaries are treated using `edge_order`++    >>> x = np.array([0, 1, 2, 3, 4])+    >>> f = x**2+    >>> np.gradient(f, edge_order=1)+    array([ 1.,  2.,  4.,  6.,  7.])+    >>> np.gradient(f, edge_order=2)+    array([-0.,  2.,  4.,  6.,  8.])++    The `axis` keyword can be used to specify a subset of axes of which the+    gradient is calculated++    >>> np.gradient(np.array([[1, 2, 6], [3, 4, 5]], dtype=float), axis=0)+    array([[ 2.,  2., -1.],+           [ 2.,  2., -1.]])++    Notes+    -----+    Assuming that :math:`f\\in C^{3}` (i.e., :math:`f` has at least 3 continuous+    derivatives) and let :math:`h_{*}` be a non-homogeneous stepsize, we+    minimize the "consistency error" :math:`\\eta_{i}` between the true gradient+    and its estimate from a linear combination of the neighboring grid-points:++    .. math::++        \\eta_{i} = f_{i}^{\\left(1\\right)} -+                    \\left[ \\alpha f\\left(x_{i}\\right) ++                            \\beta f\\left(x_{i} + h_{d}\\right) ++                            \\gamma f\\left(x_{i}-h_{s}\\right)+                    \\right]++    By substituting :math:`f(x_{i} + h_{d})` and :math:`f(x_{i} - h_{s})`+    with their Taylor series expansion, this translates into solving+    the following the linear system:++    .. math::++        \\left\\{+            \\begin{array}{r}+                \\alpha+\\beta+\\gamma=0 \\\\+                \\beta h_{d}-\\gamma h_{s}=1 \\\\+                \\beta h_{d}^{2}+\\gamma h_{s}^{2}=0+            \\end{array}+        \\right.++    The resulting approximation of :math:`f_{i}^{(1)}` is the following:++    .. math::++        \\hat f_{i}^{(1)} =+            \\frac{+                h_{s}^{2}f\\left(x_{i} + h_{d}\\right)+                + \\left(h_{d}^{2} - h_{s}^{2}\\right)f\\left(x_{i}\\right)+                - h_{d}^{2}f\\left(x_{i}-h_{s}\\right)}+                { h_{s}h_{d}\\left(h_{d} + h_{s}\\right)}+            + \\mathcal{O}\\left(\\frac{h_{d}h_{s}^{2}+                                + h_{s}h_{d}^{2}}{h_{d}+                                + h_{s}}\\right)++    It is worth noting that if :math:`h_{s}=h_{d}`+    (i.e., data are evenly spaced)+    we find the standard second order approximation:++    .. math::++        \\hat f_{i}^{(1)}=+            \\frac{f\\left(x_{i+1}\\right) - f\\left(x_{i-1}\\right)}{2h}+            + \\mathcal{O}\\left(h^{2}\\right)++    With a similar procedure the forward/backward approximations used for+    boundaries can be derived.++    References+    ----------+    .. [1]  Quarteroni A., Sacco R., Saleri F. (2007) Numerical Mathematics+            (Texts in Applied Mathematics). New York: Springer.+    .. [2]  Durran D. R. (1999) Numerical Methods for Wave Equations+            in Geophysical Fluid Dynamics. New York: Springer.+    .. [3]  Fornberg B. (1988) Generation of Finite Difference Formulas on+            Arbitrarily Spaced Grids,+            Mathematics of Computation 51, no. 184 : 699-706.+            `PDF <http://www.ams.org/journals/mcom/1988-51-184/+            S0025-5718-1988-0935077-0/S0025-5718-1988-0935077-0.pdf>`_.+    """+    f = np.asanyarray(f)+    N = f.ndim  # number of dimensions++    axes = kwargs.pop('axis', None)+    if axes is None:+        axes = tuple(range(N))+    else:+        axes = _nx.normalize_axis_tuple(axes, N)++    len_axes = len(axes)+    n = len(varargs)+    if n == 0:+        # no spacing argument - use 1 in all axes+        dx = [1.0] * len_axes+    elif n == 1 and np.ndim(varargs[0]) == 0:+        # single scalar for all axes+        dx = varargs * len_axes+    elif n == len_axes:+        # scalar or 1d array for each axis+        dx = list(varargs)+        for i, distances in enumerate(dx):+            if np.ndim(distances) == 0:+                continue+            elif np.ndim(distances) != 1:+                raise ValueError("distances must be either scalars or 1d")+            if len(distances) != f.shape[axes[i]]:+                raise ValueError("when 1d, distances must match "+                                 "the length of the corresponding dimension")+            diffx = np.diff(distances)+            # if distances are constant reduce to the scalar case+            # since it brings a consistent speedup+            if (diffx == diffx[0]).all():+                diffx = diffx[0]+            dx[i] = diffx+    else:+        raise TypeError("invalid number of arguments")++    edge_order = kwargs.pop('edge_order', 1)+    if kwargs:+        raise TypeError('"{}" are not valid keyword arguments.'.format(+                                                  '", "'.join(kwargs.keys())))+    if edge_order > 2:+        raise ValueError("'edge_order' greater than 2 not supported")++    # use central differences on interior and one-sided differences on the+    # endpoints. This preserves second order-accuracy over the full domain.++    outvals = []++    # create slice objects --- initially all are [:, :, ..., :]+    slice1 = [slice(None)]*N+    slice2 = [slice(None)]*N+    slice3 = [slice(None)]*N+    slice4 = [slice(None)]*N++    otype = f.dtype+    if otype.type is np.datetime64:+        # the timedelta dtype with the same unit information+        otype = np.dtype(otype.name.replace('datetime', 'timedelta'))+        # view as timedelta to allow addition+        f = f.view(otype)+    elif otype.type is np.timedelta64:+        pass+    elif np.issubdtype(otype, np.inexact):+        pass+    else:+        # all other types convert to floating point+        otype = np.double++    for axis, ax_dx in zip(axes, dx):+        if f.shape[axis] < edge_order + 1:+            raise ValueError(+                "Shape of array too small to calculate a numerical gradient, "+                "at least (edge_order + 1) elements are required.")+        # result allocation+        out = np.empty_like(f, dtype=otype)++        # spacing for the current axis+        uniform_spacing = np.ndim(ax_dx) == 0++        # Numerical differentiation: 2nd order interior+        slice1[axis] = slice(1, -1)+        slice2[axis] = slice(None, -2)+        slice3[axis] = slice(1, -1)+        slice4[axis] = slice(2, None)++        if uniform_spacing:+            out[tuple(slice1)] = (f[tuple(slice4)] - f[tuple(slice2)]) / (2. * ax_dx)+        else:+            dx1 = ax_dx[0:-1]+            dx2 = ax_dx[1:]+            a = -(dx2)/(dx1 * (dx1 + dx2))+            b = (dx2 - dx1) / (dx1 * dx2)+            c = dx1 / (dx2 * (dx1 + dx2))+            # fix the shape for broadcasting+            shape = np.ones(N, dtype=int)+            shape[axis] = -1+            a.shape = b.shape = c.shape = shape+            # 1D equivalent -- out[1:-1] = a * f[:-2] + b * f[1:-1] + c * f[2:]+            out[tuple(slice1)] = a * f[tuple(slice2)] + b * f[tuple(slice3)] + c * f[tuple(slice4)]++        # Numerical differentiation: 1st order edges+        if edge_order == 1:+            slice1[axis] = 0+            slice2[axis] = 1+            slice3[axis] = 0+            dx_0 = ax_dx if uniform_spacing else ax_dx[0]+            # 1D equivalent -- out[0] = (f[1] - f[0]) / (x[1] - x[0])+            out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_0++            slice1[axis] = -1+            slice2[axis] = -1+            slice3[axis] = -2+            dx_n = ax_dx if uniform_spacing else ax_dx[-1]+            # 1D equivalent -- out[-1] = (f[-1] - f[-2]) / (x[-1] - x[-2])+            out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_n++        # Numerical differentiation: 2nd order edges+        else:+            slice1[axis] = 0+            slice2[axis] = 0+            slice3[axis] = 1+            slice4[axis] = 2+            if uniform_spacing:+                a = -1.5 / ax_dx+                b = 2. / ax_dx+                c = -0.5 / ax_dx+            else:+                dx1 = ax_dx[0]+                dx2 = ax_dx[1]+                a = -(2. * dx1 + dx2)/(dx1 * (dx1 + dx2))+                b = (dx1 + dx2) / (dx1 * dx2)+                c = - dx1 / (dx2 * (dx1 + dx2))+            # 1D equivalent -- out[0] = a * f[0] + b * f[1] + c * f[2]+            out[tuple(slice1)] = a * f[tuple(slice2)] + b * f[tuple(slice3)] + c * f[tuple(slice4)]++            slice1[axis] = -1+            slice2[axis] = -3+            slice3[axis] = -2+            slice4[axis] = -1+            if uniform_spacing:+                a = 0.5 / ax_dx+                b = -2. / ax_dx+                c = 1.5 / ax_dx+            else:+                dx1 = ax_dx[-2]+                dx2 = ax_dx[-1]+                a = (dx2) / (dx1 * (dx1 + dx2))+                b = - (dx2 + dx1) / (dx1 * dx2)+                c = (2. * dx2 + dx1) / (dx2 * (dx1 + dx2))+            # 1D equivalent -- out[-1] = a * f[-3] + b * f[-2] + c * f[-1]+            out[tuple(slice1)] = a * f[tuple(slice2)] + b * f[tuple(slice3)] + c * f[tuple(slice4)]++        outvals.append(out)++        # reset the slice object in this dimension to ":"+        slice1[axis] = slice(None)+        slice2[axis] = slice(None)+        slice3[axis] = slice(None)+        slice4[axis] = slice(None)++    if len_axes == 1:+        return outvals[0]+    else:+        return outvals+++def diff(a, n=1, axis=-1):+    """+    Calculate the n-th discrete difference along the given axis.++    The first difference is given by ``out[n] = a[n+1] - a[n]`` along+    the given axis, higher differences are calculated by using `diff`+    recursively.++    Parameters+    ----------+    a : array_like+        Input array+    n : int, optional+        The number of times values are differenced. If zero, the input+        is returned as-is.+    axis : int, optional+        The axis along which the difference is taken, default is the+        last axis.++    Returns+    -------+    diff : ndarray+        The n-th differences. The shape of the output is the same as `a`+        except along `axis` where the dimension is smaller by `n`. The+        type of the output is the same as the type of the difference+        between any two elements of `a`. This is the same as the type of+        `a` in most cases. A notable exception is `datetime64`, which+        results in a `timedelta64` output array.++    See Also+    --------+    gradient, ediff1d, cumsum++    Notes+    -----+    Type is preserved for boolean arrays, so the result will contain+    `False` when consecutive elements are the same and `True` when they+    differ.++    For unsigned integer arrays, the results will also be unsigned. This+    should not be surprising, as the result is consistent with+    calculating the difference directly:++    >>> u8_arr = np.array([1, 0], dtype=np.uint8)+    >>> np.diff(u8_arr)+    array([255], dtype=uint8)+    >>> u8_arr[1,...] - u8_arr[0,...]+    array(255, np.uint8)++    If this is not desirable, then the array should be cast to a larger+    integer type first:++    >>> i16_arr = u8_arr.astype(np.int16)+    >>> np.diff(i16_arr)+    array([-1], dtype=int16)++    Examples+    --------+    >>> x = np.array([1, 2, 4, 7, 0])+    >>> np.diff(x)+    array([ 1,  2,  3, -7])+    >>> np.diff(x, n=2)+    array([  1,   1, -10])++    >>> x = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])+    >>> np.diff(x)+    array([[2, 3, 4],+           [5, 1, 2]])+    >>> np.diff(x, axis=0)+    array([[-1,  2,  0, -2]])++    >>> x = np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64)+    >>> np.diff(x)+    array([1, 1], dtype='timedelta64[D]')++    """+    if n == 0:+        return a+    if n < 0:+        raise ValueError(+            "order must be non-negative but got " + repr(n))++    a = asanyarray(a)+    nd = a.ndim+    axis = normalize_axis_index(axis, nd)++    slice1 = [slice(None)] * nd+    slice2 = [slice(None)] * nd+    slice1[axis] = slice(1, None)+    slice2[axis] = slice(None, -1)+    slice1 = tuple(slice1)+    slice2 = tuple(slice2)++    op = not_equal if a.dtype == np.bool_ else subtract+    for _ in range(n):+        a = op(a[slice1], a[slice2])++    return a+++def interp(x, xp, fp, left=None, right=None, period=None):+    """+    One-dimensional linear interpolation.++    Returns the one-dimensional piecewise linear interpolant to a function+    with given discrete data points (`xp`, `fp`), evaluated at `x`.++    Parameters+    ----------+    x : array_like+        The x-coordinates at which to evaluate the interpolated values.++    xp : 1-D sequence of floats+        The x-coordinates of the data points, must be increasing if argument+        `period` is not specified. Otherwise, `xp` is internally sorted after+        normalizing the periodic boundaries with ``xp = xp % period``.++    fp : 1-D sequence of float or complex+        The y-coordinates of the data points, same length as `xp`.++    left : optional float or complex corresponding to fp+        Value to return for `x < xp[0]`, default is `fp[0]`.++    right : optional float or complex corresponding to fp+        Value to return for `x > xp[-1]`, default is `fp[-1]`.++    period : None or float, optional+        A period for the x-coordinates. This parameter allows the proper+        interpolation of angular x-coordinates. Parameters `left` and `right`+        are ignored if `period` is specified.++        .. versionadded:: 1.10.0++    Returns+    -------+    y : float or complex (corresponding to fp) or ndarray+        The interpolated values, same shape as `x`.++    Raises+    ------+    ValueError+        If `xp` and `fp` have different length+        If `xp` or `fp` are not 1-D sequences+        If `period == 0`++    Notes+    -----+    Does not check that the x-coordinate sequence `xp` is increasing.+    If `xp` is not increasing, the results are nonsense.+    A simple check for increasing is::++        np.all(np.diff(xp) > 0)++    Examples+    --------+    >>> xp = [1, 2, 3]+    >>> fp = [3, 2, 0]+    >>> np.interp(2.5, xp, fp)+    1.0+    >>> np.interp([0, 1, 1.5, 2.72, 3.14], xp, fp)+    array([ 3. ,  3. ,  2.5 ,  0.56,  0. ])+    >>> UNDEF = -99.0+    >>> np.interp(3.14, xp, fp, right=UNDEF)+    -99.0++    Plot an interpolant to the sine function:++    >>> x = np.linspace(0, 2*np.pi, 10)+    >>> y = np.sin(x)+    >>> xvals = np.linspace(0, 2*np.pi, 50)+    >>> yinterp = np.interp(xvals, x, y)+    >>> import matplotlib.pyplot as plt+    >>> plt.plot(x, y, 'o')+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.plot(xvals, yinterp, '-x')+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.show()++    Interpolation with periodic x-coordinates:++    >>> x = [-180, -170, -185, 185, -10, -5, 0, 365]+    >>> xp = [190, -190, 350, -350]+    >>> fp = [5, 10, 3, 4]+    >>> np.interp(x, xp, fp, period=360)+    array([7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75])++    Complex interpolation:++    >>> x = [1.5, 4.0]+    >>> xp = [2,3,5]+    >>> fp = [1.0j, 0, 2+3j]+    >>> np.interp(x, xp, fp)+    array([ 0.+1.j ,  1.+1.5j])++    """++    fp = np.asarray(fp)++    if np.iscomplexobj(fp):+        interp_func = compiled_interp_complex+        input_dtype = np.complex128+    else:+        interp_func = compiled_interp+        input_dtype = np.float64++    if period is not None:+        if period == 0:+            raise ValueError("period must be a non-zero value")+        period = abs(period)+        left = None+        right = None++        x = np.asarray(x, dtype=np.float64)+        xp = np.asarray(xp, dtype=np.float64)+        fp = np.asarray(fp, dtype=input_dtype)++        if xp.ndim != 1 or fp.ndim != 1:+            raise ValueError("Data points must be 1-D sequences")+        if xp.shape[0] != fp.shape[0]:+            raise ValueError("fp and xp are not of the same length")+        # normalizing periodic boundaries+        x = x % period+        xp = xp % period+        asort_xp = np.argsort(xp)+        xp = xp[asort_xp]+        fp = fp[asort_xp]+        xp = np.concatenate((xp[-1:]-period, xp, xp[0:1]+period))+        fp = np.concatenate((fp[-1:], fp, fp[0:1]))++    return interp_func(x, xp, fp, left, right)+++def angle(z, deg=False):+    """+    Return the angle of the complex argument.++    Parameters+    ----------+    z : array_like+        A complex number or sequence of complex numbers.+    deg : bool, optional+        Return angle in degrees if True, radians if False (default).++    Returns+    -------+    angle : ndarray or scalar+        The counterclockwise angle from the positive real axis on+        the complex plane, with dtype as numpy.float64.+        +        ..versionchanged:: 1.16.0+            This function works on subclasses of ndarray like `ma.array`.++    See Also+    --------+    arctan2+    absolute++    Examples+    --------+    >>> np.angle([1.0, 1.0j, 1+1j])               # in radians+    array([ 0.        ,  1.57079633,  0.78539816])+    >>> np.angle(1+1j, deg=True)                  # in degrees+    45.0++    """+    z = asanyarray(z)+    if issubclass(z.dtype.type, _nx.complexfloating):+        zimag = z.imag+        zreal = z.real+    else:+        zimag = 0+        zreal = z++    a = arctan2(zimag, zreal)+    if deg:+        a *= 180/pi+    return a+++def unwrap(p, discont=pi, axis=-1):+    """+    Unwrap by changing deltas between values to 2*pi complement.++    Unwrap radian phase `p` by changing absolute jumps greater than+    `discont` to their 2*pi complement along the given axis.++    Parameters+    ----------+    p : array_like+        Input array.+    discont : float, optional+        Maximum discontinuity between values, default is ``pi``.+    axis : int, optional+        Axis along which unwrap will operate, default is the last axis.++    Returns+    -------+    out : ndarray+        Output array.++    See Also+    --------+    rad2deg, deg2rad++    Notes+    -----+    If the discontinuity in `p` is smaller than ``pi``, but larger than+    `discont`, no unwrapping is done because taking the 2*pi complement+    would only make the discontinuity larger.++    Examples+    --------+    >>> phase = np.linspace(0, np.pi, num=5)+    >>> phase[3:] += np.pi+    >>> phase+    array([ 0.        ,  0.78539816,  1.57079633,  5.49778714,  6.28318531])+    >>> np.unwrap(phase)+    array([ 0.        ,  0.78539816,  1.57079633, -0.78539816,  0.        ])++    """+    p = asarray(p)+    nd = p.ndim+    dd = diff(p, axis=axis)+    slice1 = [slice(None, None)]*nd     # full slices+    slice1[axis] = slice(1, None)+    slice1 = tuple(slice1)+    ddmod = mod(dd + pi, 2*pi) - pi+    _nx.copyto(ddmod, pi, where=(ddmod == -pi) & (dd > 0))+    ph_correct = ddmod - dd+    _nx.copyto(ph_correct, 0, where=abs(dd) < discont)+    up = array(p, copy=True, dtype='d')+    up[slice1] = p[slice1] + ph_correct.cumsum(axis)+    return up+++def sort_complex(a):+    """+    Sort a complex array using the real part first, then the imaginary part.++    Parameters+    ----------+    a : array_like+        Input array++    Returns+    -------+    out : complex ndarray+        Always returns a sorted complex array.++    Examples+    --------+    >>> np.sort_complex([5, 3, 6, 2, 1])+    array([ 1.+0.j,  2.+0.j,  3.+0.j,  5.+0.j,  6.+0.j])++    >>> np.sort_complex([1 + 2j, 2 - 1j, 3 - 2j, 3 - 3j, 3 + 5j])+    array([ 1.+2.j,  2.-1.j,  3.-3.j,  3.-2.j,  3.+5.j])++    """+    b = array(a, copy=True)+    b.sort()+    if not issubclass(b.dtype.type, _nx.complexfloating):+        if b.dtype.char in 'bhBH':+            return b.astype('F')+        elif b.dtype.char == 'g':+            return b.astype('G')+        else:+            return b.astype('D')+    else:+        return b+++def trim_zeros(filt, trim='fb'):+    """+    Trim the leading and/or trailing zeros from a 1-D array or sequence.++    Parameters+    ----------+    filt : 1-D array or sequence+        Input array.+    trim : str, optional+        A string with 'f' representing trim from front and 'b' to trim from+        back. Default is 'fb', trim zeros from both front and back of the+        array.++    Returns+    -------+    trimmed : 1-D array or sequence+        The result of trimming the input. The input data type is preserved.++    Examples+    --------+    >>> a = np.array((0, 0, 0, 1, 2, 3, 0, 2, 1, 0))+    >>> np.trim_zeros(a)+    array([1, 2, 3, 0, 2, 1])++    >>> np.trim_zeros(a, 'b')+    array([0, 0, 0, 1, 2, 3, 0, 2, 1])++    The input data type is preserved, list/tuple in means list/tuple out.++    >>> np.trim_zeros([0, 1, 2, 0])+    [1, 2]++    """+    first = 0+    trim = trim.upper()+    if 'F' in trim:+        for i in filt:+            if i != 0.:+                break+            else:+                first = first + 1+    last = len(filt)+    if 'B' in trim:+        for i in filt[::-1]:+            if i != 0.:+                break+            else:+                last = last - 1+    return filt[first:last]+++@deprecate+def unique(x):+    """+    This function is deprecated.  Use numpy.lib.arraysetops.unique()+    instead.+    """+    try:+        tmp = x.flatten()+        if tmp.size == 0:+            return tmp+        tmp.sort()+        idx = concatenate(([True], tmp[1:] != tmp[:-1]))+        return tmp[idx]+    except AttributeError:+        items = sorted(set(x))+        return asarray(items)+++def extract(condition, arr):+    """+    Return the elements of an array that satisfy some condition.++    This is equivalent to ``np.compress(ravel(condition), ravel(arr))``.  If+    `condition` is boolean ``np.extract`` is equivalent to ``arr[condition]``.++    Note that `place` does the exact opposite of `extract`.++    Parameters+    ----------+    condition : array_like+        An array whose nonzero or True entries indicate the elements of `arr`+        to extract.+    arr : array_like+        Input array of the same size as `condition`.++    Returns+    -------+    extract : ndarray+        Rank 1 array of values from `arr` where `condition` is True.++    See Also+    --------+    take, put, copyto, compress, place++    Examples+    --------+    >>> arr = np.arange(12).reshape((3, 4))+    >>> arr+    array([[ 0,  1,  2,  3],+           [ 4,  5,  6,  7],+           [ 8,  9, 10, 11]])+    >>> condition = np.mod(arr, 3)==0+    >>> condition+    array([[ True, False, False,  True],+           [False, False,  True, False],+           [False,  True, False, False]])+    >>> np.extract(condition, arr)+    array([0, 3, 6, 9])+++    If `condition` is boolean:++    >>> arr[condition]+    array([0, 3, 6, 9])++    """+    return _nx.take(ravel(arr), nonzero(ravel(condition))[0])+++def place(arr, mask, vals):+    """+    Change elements of an array based on conditional and input values.++    Similar to ``np.copyto(arr, vals, where=mask)``, the difference is that+    `place` uses the first N elements of `vals`, where N is the number of+    True values in `mask`, while `copyto` uses the elements where `mask`+    is True.++    Note that `extract` does the exact opposite of `place`.++    Parameters+    ----------+    arr : ndarray+        Array to put data into.+    mask : array_like+        Boolean mask array. Must have the same size as `a`.+    vals : 1-D sequence+        Values to put into `a`. Only the first N elements are used, where+        N is the number of True values in `mask`. If `vals` is smaller+        than N, it will be repeated, and if elements of `a` are to be masked,+        this sequence must be non-empty.++    See Also+    --------+    copyto, put, take, extract++    Examples+    --------+    >>> arr = np.arange(6).reshape(2, 3)+    >>> np.place(arr, arr>2, [44, 55])+    >>> arr+    array([[ 0,  1,  2],+           [44, 55, 44]])++    """+    if not isinstance(arr, np.ndarray):+        raise TypeError("argument 1 must be numpy.ndarray, "+                        "not {name}".format(name=type(arr).__name__))++    return _insert(arr, mask, vals)+++def disp(mesg, device=None, linefeed=True):+    """+    Display a message on a device.++    Parameters+    ----------+    mesg : str+        Message to display.+    device : object+        Device to write message. If None, defaults to ``sys.stdout`` which is+        very similar to ``print``. `device` needs to have ``write()`` and+        ``flush()`` methods.+    linefeed : bool, optional+        Option whether to print a line feed or not. Defaults to True.++    Raises+    ------+    AttributeError+        If `device` does not have a ``write()`` or ``flush()`` method.++    Examples+    --------+    Besides ``sys.stdout``, a file-like object can also be used as it has+    both required methods:++    >>> from io import StringIO+    >>> buf = StringIO()+    >>> np.disp(u'"Display" in a file', device=buf)+    >>> buf.getvalue()+    '"Display" in a file\\n'++    """+    if device is None:+        device = sys.stdout+    if linefeed:+        device.write('%s\n' % mesg)+    else:+        device.write('%s' % mesg)+    device.flush()+    return+++# See https://docs.scipy.org/doc/numpy/reference/c-api.generalized-ufuncs.html+_DIMENSION_NAME = r'\w+'+_CORE_DIMENSION_LIST = '(?:{0:}(?:,{0:})*)?'.format(_DIMENSION_NAME)+_ARGUMENT = r'\({}\)'.format(_CORE_DIMENSION_LIST)+_ARGUMENT_LIST = '{0:}(?:,{0:})*'.format(_ARGUMENT)+_SIGNATURE = '^{0:}->{0:}$'.format(_ARGUMENT_LIST)+++def _parse_gufunc_signature(signature):+    """+    Parse string signatures for a generalized universal function.++    Arguments+    ---------+    signature : string+        Generalized universal function signature, e.g., ``(m,n),(n,p)->(m,p)``+        for ``np.matmul``.++    Returns+    -------+    Tuple of input and output core dimensions parsed from the signature, each+    of the form List[Tuple[str, ...]].+    """+    if not re.match(_SIGNATURE, signature):+        raise ValueError(+            'not a valid gufunc signature: {}'.format(signature))+    return tuple([tuple(re.findall(_DIMENSION_NAME, arg))+                  for arg in re.findall(_ARGUMENT, arg_list)]+                 for arg_list in signature.split('->'))+++def _update_dim_sizes(dim_sizes, arg, core_dims):+    """+    Incrementally check and update core dimension sizes for a single argument.++    Arguments+    ---------+    dim_sizes : Dict[str, int]+        Sizes of existing core dimensions. Will be updated in-place.+    arg : ndarray+        Argument to examine.+    core_dims : Tuple[str, ...]+        Core dimensions for this argument.+    """+    if not core_dims:+        return++    num_core_dims = len(core_dims)+    if arg.ndim < num_core_dims:+        raise ValueError(+            '%d-dimensional argument does not have enough '+            'dimensions for all core dimensions %r'+            % (arg.ndim, core_dims))++    core_shape = arg.shape[-num_core_dims:]+    for dim, size in zip(core_dims, core_shape):+        if dim in dim_sizes:+            if size != dim_sizes[dim]:+                raise ValueError(+                    'inconsistent size for core dimension %r: %r vs %r'+                    % (dim, size, dim_sizes[dim]))+        else:+            dim_sizes[dim] = size+++def _parse_input_dimensions(args, input_core_dims):+    """+    Parse broadcast and core dimensions for vectorize with a signature.++    Arguments+    ---------+    args : Tuple[ndarray, ...]+        Tuple of input arguments to examine.+    input_core_dims : List[Tuple[str, ...]]+        List of core dimensions corresponding to each input.++    Returns+    -------+    broadcast_shape : Tuple[int, ...]+        Common shape to broadcast all non-core dimensions to.+    dim_sizes : Dict[str, int]+        Common sizes for named core dimensions.+    """+    broadcast_args = []+    dim_sizes = {}+    for arg, core_dims in zip(args, input_core_dims):+        _update_dim_sizes(dim_sizes, arg, core_dims)+        ndim = arg.ndim - len(core_dims)+        dummy_array = np.lib.stride_tricks.as_strided(0, arg.shape[:ndim])+        broadcast_args.append(dummy_array)+    broadcast_shape = np.lib.stride_tricks._broadcast_shape(*broadcast_args)+    return broadcast_shape, dim_sizes+++def _calculate_shapes(broadcast_shape, dim_sizes, list_of_core_dims):+    """Helper for calculating broadcast shapes with core dimensions."""+    return [broadcast_shape + tuple(dim_sizes[dim] for dim in core_dims)+            for core_dims in list_of_core_dims]+++def _create_arrays(broadcast_shape, dim_sizes, list_of_core_dims, dtypes):+    """Helper for creating output arrays in vectorize."""+    shapes = _calculate_shapes(broadcast_shape, dim_sizes, list_of_core_dims)+    arrays = tuple(np.empty(shape, dtype=dtype)+                   for shape, dtype in zip(shapes, dtypes))+    return arrays+++class vectorize(object):+    """+    vectorize(pyfunc, otypes=None, doc=None, excluded=None, cache=False,+              signature=None)++    Generalized function class.++    Define a vectorized function which takes a nested sequence of objects or+    numpy arrays as inputs and returns a single numpy array or a tuple of numpy+    arrays. The vectorized function evaluates `pyfunc` over successive tuples+    of the input arrays like the python map function, except it uses the+    broadcasting rules of numpy.++    The data type of the output of `vectorized` is determined by calling+    the function with the first element of the input.  This can be avoided+    by specifying the `otypes` argument.++    Parameters+    ----------+    pyfunc : callable+        A python function or method.+    otypes : str or list of dtypes, optional+        The output data type. It must be specified as either a string of+        typecode characters or a list of data type specifiers. There should+        be one data type specifier for each output.+    doc : str, optional+        The docstring for the function. If `None`, the docstring will be the+        ``pyfunc.__doc__``.+    excluded : set, optional+        Set of strings or integers representing the positional or keyword+        arguments for which the function will not be vectorized.  These will be+        passed directly to `pyfunc` unmodified.++        .. versionadded:: 1.7.0++    cache : bool, optional+       If `True`, then cache the first function call that determines the number+       of outputs if `otypes` is not provided.++        .. versionadded:: 1.7.0++    signature : string, optional+        Generalized universal function signature, e.g., ``(m,n),(n)->(m)`` for+        vectorized matrix-vector multiplication. If provided, ``pyfunc`` will+        be called with (and expected to return) arrays with shapes given by the+        size of corresponding core dimensions. By default, ``pyfunc`` is+        assumed to take scalars as input and output.++        .. versionadded:: 1.12.0++    Returns+    -------+    vectorized : callable+        Vectorized function.++    Examples+    --------+    >>> def myfunc(a, b):+    ...     "Return a-b if a>b, otherwise return a+b"+    ...     if a > b:+    ...         return a - b+    ...     else:+    ...         return a + b++    >>> vfunc = np.vectorize(myfunc)+    >>> vfunc([1, 2, 3, 4], 2)+    array([3, 4, 1, 2])++    The docstring is taken from the input function to `vectorize` unless it+    is specified:++    >>> vfunc.__doc__+    'Return a-b if a>b, otherwise return a+b'+    >>> vfunc = np.vectorize(myfunc, doc='Vectorized `myfunc`')+    >>> vfunc.__doc__+    'Vectorized `myfunc`'++    The output type is determined by evaluating the first element of the input,+    unless it is specified:++    >>> out = vfunc([1, 2, 3, 4], 2)+    >>> type(out[0])+    <type 'numpy.int32'>+    >>> vfunc = np.vectorize(myfunc, otypes=[float])+    >>> out = vfunc([1, 2, 3, 4], 2)+    >>> type(out[0])+    <type 'numpy.float64'>++    The `excluded` argument can be used to prevent vectorizing over certain+    arguments.  This can be useful for array-like arguments of a fixed length+    such as the coefficients for a polynomial as in `polyval`:++    >>> def mypolyval(p, x):+    ...     _p = list(p)+    ...     res = _p.pop(0)+    ...     while _p:+    ...         res = res*x + _p.pop(0)+    ...     return res+    >>> vpolyval = np.vectorize(mypolyval, excluded=['p'])+    >>> vpolyval(p=[1, 2, 3], x=[0, 1])+    array([3, 6])++    Positional arguments may also be excluded by specifying their position:++    >>> vpolyval.excluded.add(0)+    >>> vpolyval([1, 2, 3], x=[0, 1])+    array([3, 6])++    The `signature` argument allows for vectorizing functions that act on+    non-scalar arrays of fixed length. For example, you can use it for a+    vectorized calculation of Pearson correlation coefficient and its p-value:++    >>> import scipy.stats+    >>> pearsonr = np.vectorize(scipy.stats.pearsonr,+    ...                         signature='(n),(n)->(),()')+    >>> pearsonr([[0, 1, 2, 3]], [[1, 2, 3, 4], [4, 3, 2, 1]])+    (array([ 1., -1.]), array([ 0.,  0.]))++    Or for a vectorized convolution:++    >>> convolve = np.vectorize(np.convolve, signature='(n),(m)->(k)')+    >>> convolve(np.eye(4), [1, 2, 1])+    array([[ 1.,  2.,  1.,  0.,  0.,  0.],+           [ 0.,  1.,  2.,  1.,  0.,  0.],+           [ 0.,  0.,  1.,  2.,  1.,  0.],+           [ 0.,  0.,  0.,  1.,  2.,  1.]])++    See Also+    --------+    frompyfunc : Takes an arbitrary Python function and returns a ufunc++    Notes+    -----+    The `vectorize` function is provided primarily for convenience, not for+    performance. The implementation is essentially a for loop.++    If `otypes` is not specified, then a call to the function with the+    first argument will be used to determine the number of outputs.  The+    results of this call will be cached if `cache` is `True` to prevent+    calling the function twice.  However, to implement the cache, the+    original function must be wrapped which will slow down subsequent+    calls, so only do this if your function is expensive.++    The new keyword argument interface and `excluded` argument support+    further degrades performance.++    References+    ----------+    .. [1] NumPy Reference, section `Generalized Universal Function API+           <https://docs.scipy.org/doc/numpy/reference/c-api.generalized-ufuncs.html>`_.+    """++    def __init__(self, pyfunc, otypes=None, doc=None, excluded=None,+                 cache=False, signature=None):+        self.pyfunc = pyfunc+        self.cache = cache+        self.signature = signature+        self._ufunc = None    # Caching to improve default performance++        if doc is None:+            self.__doc__ = pyfunc.__doc__+        else:+            self.__doc__ = doc++        if isinstance(otypes, str):+            for char in otypes:+                if char not in typecodes['All']:+                    raise ValueError("Invalid otype specified: %s" % (char,))+        elif iterable(otypes):+            otypes = ''.join([_nx.dtype(x).char for x in otypes])+        elif otypes is not None:+            raise ValueError("Invalid otype specification")+        self.otypes = otypes++        # Excluded variable support+        if excluded is None:+            excluded = set()+        self.excluded = set(excluded)++        if signature is not None:+            self._in_and_out_core_dims = _parse_gufunc_signature(signature)+        else:+            self._in_and_out_core_dims = None++    def __call__(self, *args, **kwargs):+        """+        Return arrays with the results of `pyfunc` broadcast (vectorized) over+        `args` and `kwargs` not in `excluded`.+        """+        excluded = self.excluded+        if not kwargs and not excluded:+            func = self.pyfunc+            vargs = args+        else:+            # The wrapper accepts only positional arguments: we use `names` and+            # `inds` to mutate `the_args` and `kwargs` to pass to the original+            # function.+            nargs = len(args)++            names = [_n for _n in kwargs if _n not in excluded]+            inds = [_i for _i in range(nargs) if _i not in excluded]+            the_args = list(args)++            def func(*vargs):+                for _n, _i in enumerate(inds):+                    the_args[_i] = vargs[_n]+                kwargs.update(zip(names, vargs[len(inds):]))+                return self.pyfunc(*the_args, **kwargs)++            vargs = [args[_i] for _i in inds]+            vargs.extend([kwargs[_n] for _n in names])++        return self._vectorize_call(func=func, args=vargs)++    def _get_ufunc_and_otypes(self, func, args):+        """Return (ufunc, otypes)."""+        # frompyfunc will fail if args is empty+        if not args:+            raise ValueError('args can not be empty')++        if self.otypes is not None:+            otypes = self.otypes+            nout = len(otypes)++            # Note logic here: We only *use* self._ufunc if func is self.pyfunc+            # even though we set self._ufunc regardless.+            if func is self.pyfunc and self._ufunc is not None:+                ufunc = self._ufunc+            else:+                ufunc = self._ufunc = frompyfunc(func, len(args), nout)+        else:+            # Get number of outputs and output types by calling the function on+            # the first entries of args.  We also cache the result to prevent+            # the subsequent call when the ufunc is evaluated.+            # Assumes that ufunc first evaluates the 0th elements in the input+            # arrays (the input values are not checked to ensure this)+            args = [asarray(arg) for arg in args]+            if builtins.any(arg.size == 0 for arg in args):+                raise ValueError('cannot call `vectorize` on size 0 inputs '+                                 'unless `otypes` is set')++            inputs = [arg.flat[0] for arg in args]+            outputs = func(*inputs)++            # Performance note: profiling indicates that -- for simple+            # functions at least -- this wrapping can almost double the+            # execution time.+            # Hence we make it optional.+            if self.cache:+                _cache = [outputs]++                def _func(*vargs):+                    if _cache:+                        return _cache.pop()+                    else:+                        return func(*vargs)+            else:+                _func = func++            if isinstance(outputs, tuple):+                nout = len(outputs)+            else:+                nout = 1+                outputs = (outputs,)++            otypes = ''.join([asarray(outputs[_k]).dtype.char+                              for _k in range(nout)])++            # Performance note: profiling indicates that creating the ufunc is+            # not a significant cost compared with wrapping so it seems not+            # worth trying to cache this.+            ufunc = frompyfunc(_func, len(args), nout)++        return ufunc, otypes++    def _vectorize_call(self, func, args):+        """Vectorized call to `func` over positional `args`."""+        if self.signature is not None:+            res = self._vectorize_call_with_signature(func, args)+        elif not args:+            res = func()+        else:+            ufunc, otypes = self._get_ufunc_and_otypes(func=func, args=args)++            # Convert args to object arrays first+            inputs = [array(a, copy=False, subok=True, dtype=object)+                      for a in args]++            outputs = ufunc(*inputs)++            if ufunc.nout == 1:+                res = array(outputs, copy=False, subok=True, dtype=otypes[0])+            else:+                res = tuple([array(x, copy=False, subok=True, dtype=t)+                             for x, t in zip(outputs, otypes)])+        return res++    def _vectorize_call_with_signature(self, func, args):+        """Vectorized call over positional arguments with a signature."""+        input_core_dims, output_core_dims = self._in_and_out_core_dims++        if len(args) != len(input_core_dims):+            raise TypeError('wrong number of positional arguments: '+                            'expected %r, got %r'+                            % (len(input_core_dims), len(args)))+        args = tuple(asanyarray(arg) for arg in args)++        broadcast_shape, dim_sizes = _parse_input_dimensions(+            args, input_core_dims)+        input_shapes = _calculate_shapes(broadcast_shape, dim_sizes,+                                         input_core_dims)+        args = [np.broadcast_to(arg, shape, subok=True)+                for arg, shape in zip(args, input_shapes)]++        outputs = None+        otypes = self.otypes+        nout = len(output_core_dims)++        for index in np.ndindex(*broadcast_shape):+            results = func(*(arg[index] for arg in args))++            n_results = len(results) if isinstance(results, tuple) else 1++            if nout != n_results:+                raise ValueError(+                    'wrong number of outputs from pyfunc: expected %r, got %r'+                    % (nout, n_results))++            if nout == 1:+                results = (results,)++            if outputs is None:+                for result, core_dims in zip(results, output_core_dims):+                    _update_dim_sizes(dim_sizes, result, core_dims)++                if otypes is None:+                    otypes = [asarray(result).dtype for result in results]++                outputs = _create_arrays(broadcast_shape, dim_sizes,+                                         output_core_dims, otypes)++            for output, result in zip(outputs, results):+                output[index] = result++        if outputs is None:+            # did not call the function even once+            if otypes is None:+                raise ValueError('cannot call `vectorize` on size 0 inputs '+                                 'unless `otypes` is set')+            if builtins.any(dim not in dim_sizes+                            for dims in output_core_dims+                            for dim in dims):+                raise ValueError('cannot call `vectorize` with a signature '+                                 'including new output dimensions on size 0 '+                                 'inputs')+            outputs = _create_arrays(broadcast_shape, dim_sizes,+                                     output_core_dims, otypes)++        return outputs[0] if nout == 1 else outputs+++def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None,+        aweights=None):+    """+    Estimate a covariance matrix, given data and weights.++    Covariance indicates the level to which two variables vary together.+    If we examine N-dimensional samples, :math:`X = [x_1, x_2, ... x_N]^T`,+    then the covariance matrix element :math:`C_{ij}` is the covariance of+    :math:`x_i` and :math:`x_j`. The element :math:`C_{ii}` is the variance+    of :math:`x_i`.++    See the notes for an outline of the algorithm.++    Parameters+    ----------+    m : array_like+        A 1-D or 2-D array containing multiple variables and observations.+        Each row of `m` represents a variable, and each column a single+        observation of all those variables. Also see `rowvar` below.+    y : array_like, optional+        An additional set of variables and observations. `y` has the same form+        as that of `m`.+    rowvar : bool, optional+        If `rowvar` is True (default), then each row represents a+        variable, with observations in the columns. Otherwise, the relationship+        is transposed: each column represents a variable, while the rows+        contain observations.+    bias : bool, optional+        Default normalization (False) is by ``(N - 1)``, where ``N`` is the+        number of observations given (unbiased estimate). If `bias` is True,+        then normalization is by ``N``. These values can be overridden by using+        the keyword ``ddof`` in numpy versions >= 1.5.+    ddof : int, optional+        If not ``None`` the default value implied by `bias` is overridden.+        Note that ``ddof=1`` will return the unbiased estimate, even if both+        `fweights` and `aweights` are specified, and ``ddof=0`` will return+        the simple average. See the notes for the details. The default value+        is ``None``.++        .. versionadded:: 1.5+    fweights : array_like, int, optional+        1-D array of integer frequency weights; the number of times each+        observation vector should be repeated.++        .. versionadded:: 1.10+    aweights : array_like, optional+        1-D array of observation vector weights. These relative weights are+        typically large for observations considered "important" and smaller for+        observations considered less "important". If ``ddof=0`` the array of+        weights can be used to assign probabilities to observation vectors.++        .. versionadded:: 1.10++    Returns+    -------+    out : ndarray+        The covariance matrix of the variables.++    See Also+    --------+    corrcoef : Normalized covariance matrix++    Notes+    -----+    Assume that the observations are in the columns of the observation+    array `m` and let ``f = fweights`` and ``a = aweights`` for brevity. The+    steps to compute the weighted covariance are as follows::++        >>> w = f * a+        >>> v1 = np.sum(w)+        >>> v2 = np.sum(w * a)+        >>> m -= np.sum(m * w, axis=1, keepdims=True) / v1+        >>> cov = np.dot(m * w, m.T) * v1 / (v1**2 - ddof * v2)++    Note that when ``a == 1``, the normalization factor+    ``v1 / (v1**2 - ddof * v2)`` goes over to ``1 / (np.sum(f) - ddof)``+    as it should.++    Examples+    --------+    Consider two variables, :math:`x_0` and :math:`x_1`, which+    correlate perfectly, but in opposite directions:++    >>> x = np.array([[0, 2], [1, 1], [2, 0]]).T+    >>> x+    array([[0, 1, 2],+           [2, 1, 0]])++    Note how :math:`x_0` increases while :math:`x_1` decreases. The covariance+    matrix shows this clearly:++    >>> np.cov(x)+    array([[ 1., -1.],+           [-1.,  1.]])++    Note that element :math:`C_{0,1}`, which shows the correlation between+    :math:`x_0` and :math:`x_1`, is negative.++    Further, note how `x` and `y` are combined:++    >>> x = [-2.1, -1,  4.3]+    >>> y = [3,  1.1,  0.12]+    >>> X = np.stack((x, y), axis=0)+    >>> print(np.cov(X))+    [[ 11.71        -4.286     ]+     [ -4.286        2.14413333]]+    >>> print(np.cov(x, y))+    [[ 11.71        -4.286     ]+     [ -4.286        2.14413333]]+    >>> print(np.cov(x))+    11.71++    """+    # Check inputs+    if ddof is not None and ddof != int(ddof):+        raise ValueError(+            "ddof must be integer")++    # Handles complex arrays too+    m = np.asarray(m)+    if m.ndim > 2:+        raise ValueError("m has more than 2 dimensions")++    if y is None:+        dtype = np.result_type(m, np.float64)+    else:+        y = np.asarray(y)+        if y.ndim > 2:+            raise ValueError("y has more than 2 dimensions")+        dtype = np.result_type(m, y, np.float64)++    X = array(m, ndmin=2, dtype=dtype)+    if not rowvar and X.shape[0] != 1:+        X = X.T+    if X.shape[0] == 0:+        return np.array([]).reshape(0, 0)+    if y is not None:+        y = array(y, copy=False, ndmin=2, dtype=dtype)+        if not rowvar and y.shape[0] != 1:+            y = y.T+        X = np.concatenate((X, y), axis=0)++    if ddof is None:+        if bias == 0:+            ddof = 1+        else:+            ddof = 0++    # Get the product of frequencies and weights+    w = None+    if fweights is not None:+        fweights = np.asarray(fweights, dtype=float)+        if not np.all(fweights == np.around(fweights)):+            raise TypeError(+                "fweights must be integer")+        if fweights.ndim > 1:+            raise RuntimeError(+                "cannot handle multidimensional fweights")+        if fweights.shape[0] != X.shape[1]:+            raise RuntimeError(+                "incompatible numbers of samples and fweights")+        if any(fweights < 0):+            raise ValueError(+                "fweights cannot be negative")+        w = fweights+    if aweights is not None:+        aweights = np.asarray(aweights, dtype=float)+        if aweights.ndim > 1:+            raise RuntimeError(+                "cannot handle multidimensional aweights")+        if aweights.shape[0] != X.shape[1]:+            raise RuntimeError(+                "incompatible numbers of samples and aweights")+        if any(aweights < 0):+            raise ValueError(+                "aweights cannot be negative")+        if w is None:+            w = aweights+        else:+            w *= aweights++    avg, w_sum = average(X, axis=1, weights=w, returned=True)+    w_sum = w_sum[0]++    # Determine the normalization+    if w is None:+        fact = X.shape[1] - ddof+    elif ddof == 0:+        fact = w_sum+    elif aweights is None:+        fact = w_sum - ddof+    else:+        fact = w_sum - ddof*sum(w*aweights)/w_sum++    if fact <= 0:+        warnings.warn("Degrees of freedom <= 0 for slice",+                      RuntimeWarning, stacklevel=2)+        fact = 0.0++    X -= avg[:, None]+    if w is None:+        X_T = X.T+    else:+        X_T = (X*w).T+    c = dot(X, X_T.conj())+    c *= np.true_divide(1, fact)+    return c.squeeze()+++def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue):+    """+    Return Pearson product-moment correlation coefficients.++    Please refer to the documentation for `cov` for more detail.  The+    relationship between the correlation coefficient matrix, `R`, and the+    covariance matrix, `C`, is++    .. math:: R_{ij} = \\frac{ C_{ij} } { \\sqrt{ C_{ii} * C_{jj} } }++    The values of `R` are between -1 and 1, inclusive.++    Parameters+    ----------+    x : array_like+        A 1-D or 2-D array containing multiple variables and observations.+        Each row of `x` represents a variable, and each column a single+        observation of all those variables. Also see `rowvar` below.+    y : array_like, optional+        An additional set of variables and observations. `y` has the same+        shape as `x`.+    rowvar : bool, optional+        If `rowvar` is True (default), then each row represents a+        variable, with observations in the columns. Otherwise, the relationship+        is transposed: each column represents a variable, while the rows+        contain observations.+    bias : _NoValue, optional+        Has no effect, do not use.++        .. deprecated:: 1.10.0+    ddof : _NoValue, optional+        Has no effect, do not use.++        .. deprecated:: 1.10.0++    Returns+    -------+    R : ndarray+        The correlation coefficient matrix of the variables.++    See Also+    --------+    cov : Covariance matrix++    Notes+    -----+    Due to floating point rounding the resulting array may not be Hermitian,+    the diagonal elements may not be 1, and the elements may not satisfy the+    inequality abs(a) <= 1. The real and imaginary parts are clipped to the+    interval [-1,  1] in an attempt to improve on that situation but is not+    much help in the complex case.++    This function accepts but discards arguments `bias` and `ddof`.  This is+    for backwards compatibility with previous versions of this function.  These+    arguments had no effect on the return values of the function and can be+    safely ignored in this and previous versions of numpy.++    """+    if bias is not np._NoValue or ddof is not np._NoValue:+        # 2015-03-15, 1.10+        warnings.warn('bias and ddof have no effect and are deprecated',+                      DeprecationWarning, stacklevel=2)+    c = cov(x, y, rowvar)+    try:+        d = diag(c)+    except ValueError:+        # scalar covariance+        # nan if incorrect value (nan, inf, 0), 1 otherwise+        return c / c+    stddev = sqrt(d.real)+    c /= stddev[:, None]+    c /= stddev[None, :]++    # Clip real and imaginary parts to [-1, 1].  This does not guarantee+    # abs(a[i,j]) <= 1 for complex arrays, but is the best we can do without+    # excessive work.+    np.clip(c.real, -1, 1, out=c.real)+    if np.iscomplexobj(c):+        np.clip(c.imag, -1, 1, out=c.imag)++    return c+++def blackman(M):+    """+    Return the Blackman window.++    The Blackman window is a taper formed by using the first three+    terms of a summation of cosines. It was designed to have close to the+    minimal leakage possible.  It is close to optimal, only slightly worse+    than a Kaiser window.++    Parameters+    ----------+    M : int+        Number of points in the output window. If zero or less, an empty+        array is returned.++    Returns+    -------+    out : ndarray+        The window, with the maximum value normalized to one (the value one+        appears only if the number of samples is odd).++    See Also+    --------+    bartlett, hamming, hanning, kaiser++    Notes+    -----+    The Blackman window is defined as++    .. math::  w(n) = 0.42 - 0.5 \\cos(2\\pi n/M) + 0.08 \\cos(4\\pi n/M)++    Most references to the Blackman window come from the signal processing+    literature, where it is used as one of many windowing functions for+    smoothing values.  It is also known as an apodization (which means+    "removing the foot", i.e. smoothing discontinuities at the beginning+    and end of the sampled signal) or tapering function. It is known as a+    "near optimal" tapering function, almost as good (by some measures)+    as the kaiser window.++    References+    ----------+    Blackman, R.B. and Tukey, J.W., (1958) The measurement of power spectra,+    Dover Publications, New York.++    Oppenheim, A.V., and R.W. Schafer. Discrete-Time Signal Processing.+    Upper Saddle River, NJ: Prentice-Hall, 1999, pp. 468-471.++    Examples+    --------+    >>> np.blackman(12)+    array([ -1.38777878e-17,   3.26064346e-02,   1.59903635e-01,+             4.14397981e-01,   7.36045180e-01,   9.67046769e-01,+             9.67046769e-01,   7.36045180e-01,   4.14397981e-01,+             1.59903635e-01,   3.26064346e-02,  -1.38777878e-17])+++    Plot the window and the frequency response:++    >>> from numpy.fft import fft, fftshift+    >>> window = np.blackman(51)+    >>> plt.plot(window)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Blackman window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Amplitude")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Sample")+    <matplotlib.text.Text object at 0x...>+    >>> plt.show()++    >>> plt.figure()+    <matplotlib.figure.Figure object at 0x...>+    >>> A = fft(window, 2048) / 25.5+    >>> mag = np.abs(fftshift(A))+    >>> freq = np.linspace(-0.5, 0.5, len(A))+    >>> response = 20 * np.log10(mag)+    >>> response = np.clip(response, -100, 100)+    >>> plt.plot(freq, response)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Frequency response of Blackman window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Magnitude [dB]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Normalized frequency [cycles per sample]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.axis('tight')+    (-0.5, 0.5, -100.0, ...)+    >>> plt.show()++    """+    if M < 1:+        return array([])+    if M == 1:+        return ones(1, float)+    n = arange(0, M)+    return 0.42 - 0.5*cos(2.0*pi*n/(M-1)) + 0.08*cos(4.0*pi*n/(M-1))+++def bartlett(M):+    """+    Return the Bartlett window.++    The Bartlett window is very similar to a triangular window, except+    that the end points are at zero.  It is often used in signal+    processing for tapering a signal, without generating too much+    ripple in the frequency domain.++    Parameters+    ----------+    M : int+        Number of points in the output window. If zero or less, an+        empty array is returned.++    Returns+    -------+    out : array+        The triangular window, with the maximum value normalized to one+        (the value one appears only if the number of samples is odd), with+        the first and last samples equal to zero.++    See Also+    --------+    blackman, hamming, hanning, kaiser++    Notes+    -----+    The Bartlett window is defined as++    .. math:: w(n) = \\frac{2}{M-1} \\left(+              \\frac{M-1}{2} - \\left|n - \\frac{M-1}{2}\\right|+              \\right)++    Most references to the Bartlett window come from the signal+    processing literature, where it is used as one of many windowing+    functions for smoothing values.  Note that convolution with this+    window produces linear interpolation.  It is also known as an+    apodization (which means"removing the foot", i.e. smoothing+    discontinuities at the beginning and end of the sampled signal) or+    tapering function. The fourier transform of the Bartlett is the product+    of two sinc functions.+    Note the excellent discussion in Kanasewich.++    References+    ----------+    .. [1] M.S. Bartlett, "Periodogram Analysis and Continuous Spectra",+           Biometrika 37, 1-16, 1950.+    .. [2] E.R. Kanasewich, "Time Sequence Analysis in Geophysics",+           The University of Alberta Press, 1975, pp. 109-110.+    .. [3] A.V. Oppenheim and R.W. Schafer, "Discrete-Time Signal+           Processing", Prentice-Hall, 1999, pp. 468-471.+    .. [4] Wikipedia, "Window function",+           https://en.wikipedia.org/wiki/Window_function+    .. [5] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,+           "Numerical Recipes", Cambridge University Press, 1986, page 429.++    Examples+    --------+    >>> np.bartlett(12)+    array([ 0.        ,  0.18181818,  0.36363636,  0.54545455,  0.72727273,+            0.90909091,  0.90909091,  0.72727273,  0.54545455,  0.36363636,+            0.18181818,  0.        ])++    Plot the window and its frequency response (requires SciPy and matplotlib):++    >>> from numpy.fft import fft, fftshift+    >>> window = np.bartlett(51)+    >>> plt.plot(window)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Bartlett window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Amplitude")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Sample")+    <matplotlib.text.Text object at 0x...>+    >>> plt.show()++    >>> plt.figure()+    <matplotlib.figure.Figure object at 0x...>+    >>> A = fft(window, 2048) / 25.5+    >>> mag = np.abs(fftshift(A))+    >>> freq = np.linspace(-0.5, 0.5, len(A))+    >>> response = 20 * np.log10(mag)+    >>> response = np.clip(response, -100, 100)+    >>> plt.plot(freq, response)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Frequency response of Bartlett window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Magnitude [dB]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Normalized frequency [cycles per sample]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.axis('tight')+    (-0.5, 0.5, -100.0, ...)+    >>> plt.show()++    """+    if M < 1:+        return array([])+    if M == 1:+        return ones(1, float)+    n = arange(0, M)+    return where(less_equal(n, (M-1)/2.0), 2.0*n/(M-1), 2.0 - 2.0*n/(M-1))+++def hanning(M):+    """+    Return the Hanning window.++    The Hanning window is a taper formed by using a weighted cosine.++    Parameters+    ----------+    M : int+        Number of points in the output window. If zero or less, an+        empty array is returned.++    Returns+    -------+    out : ndarray, shape(M,)+        The window, with the maximum value normalized to one (the value+        one appears only if `M` is odd).++    See Also+    --------+    bartlett, blackman, hamming, kaiser++    Notes+    -----+    The Hanning window is defined as++    .. math::  w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right)+               \\qquad 0 \\leq n \\leq M-1++    The Hanning was named for Julius von Hann, an Austrian meteorologist.+    It is also known as the Cosine Bell. Some authors prefer that it be+    called a Hann window, to help avoid confusion with the very similar+    Hamming window.++    Most references to the Hanning window come from the signal processing+    literature, where it is used as one of many windowing functions for+    smoothing values.  It is also known as an apodization (which means+    "removing the foot", i.e. smoothing discontinuities at the beginning+    and end of the sampled signal) or tapering function.++    References+    ----------+    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power+           spectra, Dover Publications, New York.+    .. [2] E.R. Kanasewich, "Time Sequence Analysis in Geophysics",+           The University of Alberta Press, 1975, pp. 106-108.+    .. [3] Wikipedia, "Window function",+           https://en.wikipedia.org/wiki/Window_function+    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,+           "Numerical Recipes", Cambridge University Press, 1986, page 425.++    Examples+    --------+    >>> np.hanning(12)+    array([ 0.        ,  0.07937323,  0.29229249,  0.57115742,  0.82743037,+            0.97974649,  0.97974649,  0.82743037,  0.57115742,  0.29229249,+            0.07937323,  0.        ])++    Plot the window and its frequency response:++    >>> from numpy.fft import fft, fftshift+    >>> window = np.hanning(51)+    >>> plt.plot(window)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Hann window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Amplitude")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Sample")+    <matplotlib.text.Text object at 0x...>+    >>> plt.show()++    >>> plt.figure()+    <matplotlib.figure.Figure object at 0x...>+    >>> A = fft(window, 2048) / 25.5+    >>> mag = np.abs(fftshift(A))+    >>> freq = np.linspace(-0.5, 0.5, len(A))+    >>> response = 20 * np.log10(mag)+    >>> response = np.clip(response, -100, 100)+    >>> plt.plot(freq, response)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Frequency response of the Hann window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Magnitude [dB]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Normalized frequency [cycles per sample]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.axis('tight')+    (-0.5, 0.5, -100.0, ...)+    >>> plt.show()++    """+    if M < 1:+        return array([])+    if M == 1:+        return ones(1, float)+    n = arange(0, M)+    return 0.5 - 0.5*cos(2.0*pi*n/(M-1))+++def hamming(M):+    """+    Return the Hamming window.++    The Hamming window is a taper formed by using a weighted cosine.++    Parameters+    ----------+    M : int+        Number of points in the output window. If zero or less, an+        empty array is returned.++    Returns+    -------+    out : ndarray+        The window, with the maximum value normalized to one (the value+        one appears only if the number of samples is odd).++    See Also+    --------+    bartlett, blackman, hanning, kaiser++    Notes+    -----+    The Hamming window is defined as++    .. math::  w(n) = 0.54 - 0.46cos\\left(\\frac{2\\pi{n}}{M-1}\\right)+               \\qquad 0 \\leq n \\leq M-1++    The Hamming was named for R. W. Hamming, an associate of J. W. Tukey+    and is described in Blackman and Tukey. It was recommended for+    smoothing the truncated autocovariance function in the time domain.+    Most references to the Hamming window come from the signal processing+    literature, where it is used as one of many windowing functions for+    smoothing values.  It is also known as an apodization (which means+    "removing the foot", i.e. smoothing discontinuities at the beginning+    and end of the sampled signal) or tapering function.++    References+    ----------+    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power+           spectra, Dover Publications, New York.+    .. [2] E.R. Kanasewich, "Time Sequence Analysis in Geophysics", The+           University of Alberta Press, 1975, pp. 109-110.+    .. [3] Wikipedia, "Window function",+           https://en.wikipedia.org/wiki/Window_function+    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,+           "Numerical Recipes", Cambridge University Press, 1986, page 425.++    Examples+    --------+    >>> np.hamming(12)+    array([ 0.08      ,  0.15302337,  0.34890909,  0.60546483,  0.84123594,+            0.98136677,  0.98136677,  0.84123594,  0.60546483,  0.34890909,+            0.15302337,  0.08      ])++    Plot the window and the frequency response:++    >>> from numpy.fft import fft, fftshift+    >>> window = np.hamming(51)+    >>> plt.plot(window)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Hamming window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Amplitude")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Sample")+    <matplotlib.text.Text object at 0x...>+    >>> plt.show()++    >>> plt.figure()+    <matplotlib.figure.Figure object at 0x...>+    >>> A = fft(window, 2048) / 25.5+    >>> mag = np.abs(fftshift(A))+    >>> freq = np.linspace(-0.5, 0.5, len(A))+    >>> response = 20 * np.log10(mag)+    >>> response = np.clip(response, -100, 100)+    >>> plt.plot(freq, response)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Frequency response of Hamming window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Magnitude [dB]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Normalized frequency [cycles per sample]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.axis('tight')+    (-0.5, 0.5, -100.0, ...)+    >>> plt.show()++    """+    if M < 1:+        return array([])+    if M == 1:+        return ones(1, float)+    n = arange(0, M)+    return 0.54 - 0.46*cos(2.0*pi*n/(M-1))++## Code from cephes for i0++_i0A = [+    -4.41534164647933937950E-18,+    3.33079451882223809783E-17,+    -2.43127984654795469359E-16,+    1.71539128555513303061E-15,+    -1.16853328779934516808E-14,+    7.67618549860493561688E-14,+    -4.85644678311192946090E-13,+    2.95505266312963983461E-12,+    -1.72682629144155570723E-11,+    9.67580903537323691224E-11,+    -5.18979560163526290666E-10,+    2.65982372468238665035E-9,+    -1.30002500998624804212E-8,+    6.04699502254191894932E-8,+    -2.67079385394061173391E-7,+    1.11738753912010371815E-6,+    -4.41673835845875056359E-6,+    1.64484480707288970893E-5,+    -5.75419501008210370398E-5,+    1.88502885095841655729E-4,+    -5.76375574538582365885E-4,+    1.63947561694133579842E-3,+    -4.32430999505057594430E-3,+    1.05464603945949983183E-2,+    -2.37374148058994688156E-2,+    4.93052842396707084878E-2,+    -9.49010970480476444210E-2,+    1.71620901522208775349E-1,+    -3.04682672343198398683E-1,+    6.76795274409476084995E-1+    ]++_i0B = [+    -7.23318048787475395456E-18,+    -4.83050448594418207126E-18,+    4.46562142029675999901E-17,+    3.46122286769746109310E-17,+    -2.82762398051658348494E-16,+    -3.42548561967721913462E-16,+    1.77256013305652638360E-15,+    3.81168066935262242075E-15,+    -9.55484669882830764870E-15,+    -4.15056934728722208663E-14,+    1.54008621752140982691E-14,+    3.85277838274214270114E-13,+    7.18012445138366623367E-13,+    -1.79417853150680611778E-12,+    -1.32158118404477131188E-11,+    -3.14991652796324136454E-11,+    1.18891471078464383424E-11,+    4.94060238822496958910E-10,+    3.39623202570838634515E-9,+    2.26666899049817806459E-8,+    2.04891858946906374183E-7,+    2.89137052083475648297E-6,+    6.88975834691682398426E-5,+    3.36911647825569408990E-3,+    8.04490411014108831608E-1+    ]+++def _chbevl(x, vals):+    b0 = vals[0]+    b1 = 0.0++    for i in range(1, len(vals)):+        b2 = b1+        b1 = b0+        b0 = x*b1 - b2 + vals[i]++    return 0.5*(b0 - b2)+++def _i0_1(x):+    return exp(x) * _chbevl(x/2.0-2, _i0A)+++def _i0_2(x):+    return exp(x) * _chbevl(32.0/x - 2.0, _i0B) / sqrt(x)+++def i0(x):+    """+    Modified Bessel function of the first kind, order 0.++    Usually denoted :math:`I_0`.  This function does broadcast, but will *not*+    "up-cast" int dtype arguments unless accompanied by at least one float or+    complex dtype argument (see Raises below).++    Parameters+    ----------+    x : array_like, dtype float or complex+        Argument of the Bessel function.++    Returns+    -------+    out : ndarray, shape = x.shape, dtype = x.dtype+        The modified Bessel function evaluated at each of the elements of `x`.++    Raises+    ------+    TypeError: array cannot be safely cast to required type+        If argument consists exclusively of int dtypes.++    See Also+    --------+    scipy.special.iv, scipy.special.ive++    Notes+    -----+    We use the algorithm published by Clenshaw [1]_ and referenced by+    Abramowitz and Stegun [2]_, for which the function domain is+    partitioned into the two intervals [0,8] and (8,inf), and Chebyshev+    polynomial expansions are employed in each interval. Relative error on+    the domain [0,30] using IEEE arithmetic is documented [3]_ as having a+    peak of 5.8e-16 with an rms of 1.4e-16 (n = 30000).++    References+    ----------+    .. [1] C. W. Clenshaw, "Chebyshev series for mathematical functions", in+           *National Physical Laboratory Mathematical Tables*, vol. 5, London:+           Her Majesty's Stationery Office, 1962.+    .. [2] M. Abramowitz and I. A. Stegun, *Handbook of Mathematical+           Functions*, 10th printing, New York: Dover, 1964, pp. 379.+           http://www.math.sfu.ca/~cbm/aands/page_379.htm+    .. [3] http://kobesearch.cpan.org/htdocs/Math-Cephes/Math/Cephes.html++    Examples+    --------+    >>> np.i0([0.])+    array(1.0)+    >>> np.i0([0., 1. + 2j])+    array([ 1.00000000+0.j        ,  0.18785373+0.64616944j])++    """+    x = atleast_1d(x).copy()+    y = empty_like(x)+    ind = (x < 0)+    x[ind] = -x[ind]+    ind = (x <= 8.0)+    y[ind] = _i0_1(x[ind])+    ind2 = ~ind+    y[ind2] = _i0_2(x[ind2])+    return y.squeeze()++## End of cephes code for i0+++def kaiser(M, beta):+    """+    Return the Kaiser window.++    The Kaiser window is a taper formed by using a Bessel function.++    Parameters+    ----------+    M : int+        Number of points in the output window. If zero or less, an+        empty array is returned.+    beta : float+        Shape parameter for window.++    Returns+    -------+    out : array+        The window, with the maximum value normalized to one (the value+        one appears only if the number of samples is odd).++    See Also+    --------+    bartlett, blackman, hamming, hanning++    Notes+    -----+    The Kaiser window is defined as++    .. math::  w(n) = I_0\\left( \\beta \\sqrt{1-\\frac{4n^2}{(M-1)^2}}+               \\right)/I_0(\\beta)++    with++    .. math:: \\quad -\\frac{M-1}{2} \\leq n \\leq \\frac{M-1}{2},++    where :math:`I_0` is the modified zeroth-order Bessel function.++    The Kaiser was named for Jim Kaiser, who discovered a simple+    approximation to the DPSS window based on Bessel functions.  The Kaiser+    window is a very good approximation to the Digital Prolate Spheroidal+    Sequence, or Slepian window, which is the transform which maximizes the+    energy in the main lobe of the window relative to total energy.++    The Kaiser can approximate many other windows by varying the beta+    parameter.++    ====  =======================+    beta  Window shape+    ====  =======================+    0     Rectangular+    5     Similar to a Hamming+    6     Similar to a Hanning+    8.6   Similar to a Blackman+    ====  =======================++    A beta value of 14 is probably a good starting point. Note that as beta+    gets large, the window narrows, and so the number of samples needs to be+    large enough to sample the increasingly narrow spike, otherwise NaNs will+    get returned.++    Most references to the Kaiser window come from the signal processing+    literature, where it is used as one of many windowing functions for+    smoothing values.  It is also known as an apodization (which means+    "removing the foot", i.e. smoothing discontinuities at the beginning+    and end of the sampled signal) or tapering function.++    References+    ----------+    .. [1] J. F. Kaiser, "Digital Filters" - Ch 7 in "Systems analysis by+           digital computer", Editors: F.F. Kuo and J.F. Kaiser, p 218-285.+           John Wiley and Sons, New York, (1966).+    .. [2] E.R. Kanasewich, "Time Sequence Analysis in Geophysics", The+           University of Alberta Press, 1975, pp. 177-178.+    .. [3] Wikipedia, "Window function",+           https://en.wikipedia.org/wiki/Window_function++    Examples+    --------+    >>> np.kaiser(12, 14)+    array([  7.72686684e-06,   3.46009194e-03,   4.65200189e-02,+             2.29737120e-01,   5.99885316e-01,   9.45674898e-01,+             9.45674898e-01,   5.99885316e-01,   2.29737120e-01,+             4.65200189e-02,   3.46009194e-03,   7.72686684e-06])+++    Plot the window and the frequency response:++    >>> from numpy.fft import fft, fftshift+    >>> window = np.kaiser(51, 14)+    >>> plt.plot(window)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Kaiser window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Amplitude")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Sample")+    <matplotlib.text.Text object at 0x...>+    >>> plt.show()++    >>> plt.figure()+    <matplotlib.figure.Figure object at 0x...>+    >>> A = fft(window, 2048) / 25.5+    >>> mag = np.abs(fftshift(A))+    >>> freq = np.linspace(-0.5, 0.5, len(A))+    >>> response = 20 * np.log10(mag)+    >>> response = np.clip(response, -100, 100)+    >>> plt.plot(freq, response)+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Frequency response of Kaiser window")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Magnitude [dB]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("Normalized frequency [cycles per sample]")+    <matplotlib.text.Text object at 0x...>+    >>> plt.axis('tight')+    (-0.5, 0.5, -100.0, ...)+    >>> plt.show()++    """+    from numpy.dual import i0+    if M == 1:+        return np.array([1.])+    n = arange(0, M)+    alpha = (M-1)/2.0+    return i0(beta * sqrt(1-((n-alpha)/alpha)**2.0))/i0(float(beta))+++def sinc(x):+    """+    Return the sinc function.++    The sinc function is :math:`\\sin(\\pi x)/(\\pi x)`.++    Parameters+    ----------+    x : ndarray+        Array (possibly multi-dimensional) of values for which to to+        calculate ``sinc(x)``.++    Returns+    -------+    out : ndarray+        ``sinc(x)``, which has the same shape as the input.++    Notes+    -----+    ``sinc(0)`` is the limit value 1.++    The name sinc is short for "sine cardinal" or "sinus cardinalis".++    The sinc function is used in various signal processing applications,+    including in anti-aliasing, in the construction of a Lanczos resampling+    filter, and in interpolation.++    For bandlimited interpolation of discrete-time signals, the ideal+    interpolation kernel is proportional to the sinc function.++    References+    ----------+    .. [1] Weisstein, Eric W. "Sinc Function." From MathWorld--A Wolfram Web+           Resource. http://mathworld.wolfram.com/SincFunction.html+    .. [2] Wikipedia, "Sinc function",+           https://en.wikipedia.org/wiki/Sinc_function++    Examples+    --------+    >>> x = np.linspace(-4, 4, 41)+    >>> np.sinc(x)+    array([ -3.89804309e-17,  -4.92362781e-02,  -8.40918587e-02,+            -8.90384387e-02,  -5.84680802e-02,   3.89804309e-17,+             6.68206631e-02,   1.16434881e-01,   1.26137788e-01,+             8.50444803e-02,  -3.89804309e-17,  -1.03943254e-01,+            -1.89206682e-01,  -2.16236208e-01,  -1.55914881e-01,+             3.89804309e-17,   2.33872321e-01,   5.04551152e-01,+             7.56826729e-01,   9.35489284e-01,   1.00000000e+00,+             9.35489284e-01,   7.56826729e-01,   5.04551152e-01,+             2.33872321e-01,   3.89804309e-17,  -1.55914881e-01,+            -2.16236208e-01,  -1.89206682e-01,  -1.03943254e-01,+            -3.89804309e-17,   8.50444803e-02,   1.26137788e-01,+             1.16434881e-01,   6.68206631e-02,   3.89804309e-17,+            -5.84680802e-02,  -8.90384387e-02,  -8.40918587e-02,+            -4.92362781e-02,  -3.89804309e-17])++    >>> plt.plot(x, np.sinc(x))+    [<matplotlib.lines.Line2D object at 0x...>]+    >>> plt.title("Sinc Function")+    <matplotlib.text.Text object at 0x...>+    >>> plt.ylabel("Amplitude")+    <matplotlib.text.Text object at 0x...>+    >>> plt.xlabel("X")+    <matplotlib.text.Text object at 0x...>+    >>> plt.show()++    It works in 2-D as well:++    >>> x = np.linspace(-4, 4, 401)+    >>> xx = np.outer(x, x)+    >>> plt.imshow(np.sinc(xx))+    <matplotlib.image.AxesImage object at 0x...>++    """+    x = np.asanyarray(x)+    y = pi * where(x == 0, 1.0e-20, x)+    return sin(y)/y+++def msort(a):+    """+    Return a copy of an array sorted along the first axis.++    Parameters+    ----------+    a : array_like+        Array to be sorted.++    Returns+    -------+    sorted_array : ndarray+        Array of the same type and shape as `a`.++    See Also+    --------+    sort++    Notes+    -----+    ``np.msort(a)`` is equivalent to  ``np.sort(a, axis=0)``.++    """+    b = array(a, subok=True, copy=True)+    b.sort(0)+    return b+++def _ureduce(a, func, **kwargs):+    """+    Internal Function.+    Call `func` with `a` as first argument swapping the axes to use extended+    axis on functions that don't support it natively.++    Returns result and a.shape with axis dims set to 1.++    Parameters+    ----------+    a : array_like+        Input array or object that can be converted to an array.+    func : callable+        Reduction function capable of receiving a single axis argument.+        It is called with `a` as first argument followed by `kwargs`.+    kwargs : keyword arguments+        additional keyword arguments to pass to `func`.++    Returns+    -------+    result : tuple+        Result of func(a, **kwargs) and a.shape with axis dims set to 1+        which can be used to reshape the result to the same shape a ufunc with+        keepdims=True would produce.++    """+    a = np.asanyarray(a)+    axis = kwargs.get('axis', None)+    if axis is not None:+        keepdim = list(a.shape)+        nd = a.ndim+        axis = _nx.normalize_axis_tuple(axis, nd)++        for ax in axis:+            keepdim[ax] = 1++        if len(axis) == 1:+            kwargs['axis'] = axis[0]+        else:+            keep = set(range(nd)) - set(axis)+            nkeep = len(keep)+            # swap axis that should not be reduced to front+            for i, s in enumerate(sorted(keep)):+                a = a.swapaxes(i, s)+            # merge reduced axis+            a = a.reshape(a.shape[:nkeep] + (-1,))+            kwargs['axis'] = -1+        keepdim = tuple(keepdim)+    else:+        keepdim = (1,) * a.ndim++    r = func(a, **kwargs)+    return r, keepdim+++def median(a, axis=None, out=None, overwrite_input=False, keepdims=False):+    """+    Compute the median along the specified axis.++    Returns the median of the array elements.++    Parameters+    ----------+    a : array_like+        Input array or object that can be converted to an array.+    axis : {int, sequence of int, None}, optional+        Axis or axes along which the medians are computed. The default+        is to compute the median along a flattened version of the array.+        A sequence of axes is supported since version 1.9.0.+    out : ndarray, optional+        Alternative output array in which to place the result. It must+        have the same shape and buffer length as the expected output,+        but the type (of the output) will be cast if necessary.+    overwrite_input : bool, optional+       If True, then allow use of memory of input array `a` for+       calculations. The input array will be modified by the call to+       `median`. This will save memory when you do not need to preserve+       the contents of the input array. Treat the input as undefined,+       but it will probably be fully or partially sorted. Default is+       False. If `overwrite_input` is ``True`` and `a` is not already an+       `ndarray`, an error will be raised.+    keepdims : bool, optional+        If this is set to True, the axes which are reduced are left+        in the result as dimensions with size one. With this option,+        the result will broadcast correctly against the original `arr`.++        .. versionadded:: 1.9.0++    Returns+    -------+    median : ndarray+        A new array holding the result. If the input contains integers+        or floats smaller than ``float64``, then the output data-type is+        ``np.float64``.  Otherwise, the data-type of the output is the+        same as that of the input. If `out` is specified, that array is+        returned instead.++    See Also+    --------+    mean, percentile++    Notes+    -----+    Given a vector ``V`` of length ``N``, the median of ``V`` is the+    middle value of a sorted copy of ``V``, ``V_sorted`` - i+    e., ``V_sorted[(N-1)/2]``, when ``N`` is odd, and the average of the+    two middle values of ``V_sorted`` when ``N`` is even.++    Examples+    --------+    >>> a = np.array([[10, 7, 4], [3, 2, 1]])+    >>> a+    array([[10,  7,  4],+           [ 3,  2,  1]])+    >>> np.median(a)+    3.5+    >>> np.median(a, axis=0)+    array([ 6.5,  4.5,  2.5])+    >>> np.median(a, axis=1)+    array([ 7.,  2.])+    >>> m = np.median(a, axis=0)+    >>> out = np.zeros_like(m)+    >>> np.median(a, axis=0, out=m)+    array([ 6.5,  4.5,  2.5])+    >>> m+    array([ 6.5,  4.5,  2.5])+    >>> b = a.copy()+    >>> np.median(b, axis=1, overwrite_input=True)+    array([ 7.,  2.])+    >>> assert not np.all(a==b)+    >>> b = a.copy()+    >>> np.median(b, axis=None, overwrite_input=True)+    3.5+    >>> assert not np.all(a==b)++    """+    r, k = _ureduce(a, func=_median, axis=axis, out=out,+                    overwrite_input=overwrite_input)+    if keepdims:+        return r.reshape(k)+    else:+        return r++def _median(a, axis=None, out=None, overwrite_input=False):+    # can't be reasonably be implemented in terms of percentile as we have to+    # call mean to not break astropy+    a = np.asanyarray(a)++    # Set the partition indexes+    if axis is None:+        sz = a.size+    else:+        sz = a.shape[axis]+    if sz % 2 == 0:+        szh = sz // 2+        kth = [szh - 1, szh]+    else:+        kth = [(sz - 1) // 2]+    # Check if the array contains any nan's+    if np.issubdtype(a.dtype, np.inexact):+        kth.append(-1)++    if overwrite_input:+        if axis is None:+            part = a.ravel()+            part.partition(kth)+        else:+            a.partition(kth, axis=axis)+            part = a+    else:+        part = partition(a, kth, axis=axis)++    if part.shape == ():+        # make 0-D arrays work+        return part.item()+    if axis is None:+        axis = 0++    indexer = [slice(None)] * part.ndim+    index = part.shape[axis] // 2+    if part.shape[axis] % 2 == 1:+        # index with slice to allow mean (below) to work+        indexer[axis] = slice(index, index+1)+    else:+        indexer[axis] = slice(index-1, index+1)+    indexer = tuple(indexer)++    # Check if the array contains any nan's+    if np.issubdtype(a.dtype, np.inexact) and sz > 0:+        # warn and return nans like mean would+        rout = mean(part[indexer], axis=axis, out=out)+        return np.lib.utils._median_nancheck(part, rout, axis, out)+    else:+        # if there are no nans+        # Use mean in odd and even case to coerce data type+        # and check, use out array.+        return mean(part[indexer], axis=axis, out=out)+++def percentile(a, q, axis=None, out=None,+               overwrite_input=False, interpolation='linear', keepdims=False):+    """+    Compute the q-th percentile of the data along the specified axis.++    Returns the q-th percentile(s) of the array elements.++    Parameters+    ----------+    a : array_like+        Input array or object that can be converted to an array.+    q : array_like of float+        Percentile or sequence of percentiles to compute, which must be between+        0 and 100 inclusive.+    axis : {int, tuple of int, None}, optional+        Axis or axes along which the percentiles are computed. The+        default is to compute the percentile(s) along a flattened+        version of the array.++        .. versionchanged:: 1.9.0+            A tuple of axes is supported+    out : ndarray, optional+        Alternative output array in which to place the result. It must+        have the same shape and buffer length as the expected output,+        but the type (of the output) will be cast if necessary.+    overwrite_input : bool, optional+        If True, then allow the input array `a` to be modified by intermediate+        calculations, to save memory. In this case, the contents of the input+        `a` after this function completes is undefined.++    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}+        This optional parameter specifies the interpolation method to+        use when the desired percentile lies between two data points+        ``i < j``:++        * 'linear': ``i + (j - i) * fraction``, where ``fraction``+          is the fractional part of the index surrounded by ``i``+          and ``j``.+        * 'lower': ``i``.+        * 'higher': ``j``.+        * 'nearest': ``i`` or ``j``, whichever is nearest.+        * 'midpoint': ``(i + j) / 2``.++        .. versionadded:: 1.9.0+    keepdims : bool, optional+        If this is set to True, the axes which are reduced are left in+        the result as dimensions with size one. With this option, the+        result will broadcast correctly against the original array `a`.++        .. versionadded:: 1.9.0++    Returns+    -------+    percentile : scalar or ndarray+        If `q` is a single percentile and `axis=None`, then the result+        is a scalar. If multiple percentiles are given, first axis of+        the result corresponds to the percentiles. The other axes are+        the axes that remain after the reduction of `a`. If the input+        contains integers or floats smaller than ``float64``, the output+        data-type is ``float64``. Otherwise, the output data-type is the+        same as that of the input. If `out` is specified, that array is+        returned instead.++    See Also+    --------+    mean+    median : equivalent to ``percentile(..., 50)``+    nanpercentile+    quantile : equivalent to percentile, except with q in the range [0, 1].++    Notes+    -----+    Given a vector ``V`` of length ``N``, the q-th percentile of+    ``V`` is the value ``q/100`` of the way from the minimum to the+    maximum in a sorted copy of ``V``. The values and distances of+    the two nearest neighbors as well as the `interpolation` parameter+    will determine the percentile if the normalized ranking does not+    match the location of ``q`` exactly. This function is the same as+    the median if ``q=50``, the same as the minimum if ``q=0`` and the+    same as the maximum if ``q=100``.++    Examples+    --------+    >>> a = np.array([[10, 7, 4], [3, 2, 1]])+    >>> a+    array([[10,  7,  4],+           [ 3,  2,  1]])+    >>> np.percentile(a, 50)+    3.5+    >>> np.percentile(a, 50, axis=0)+    array([[ 6.5,  4.5,  2.5]])+    >>> np.percentile(a, 50, axis=1)+    array([ 7.,  2.])+    >>> np.percentile(a, 50, axis=1, keepdims=True)+    array([[ 7.],+           [ 2.]])++    >>> m = np.percentile(a, 50, axis=0)+    >>> out = np.zeros_like(m)+    >>> np.percentile(a, 50, axis=0, out=out)+    array([[ 6.5,  4.5,  2.5]])+    >>> m+    array([[ 6.5,  4.5,  2.5]])++    >>> b = a.copy()+    >>> np.percentile(b, 50, axis=1, overwrite_input=True)+    array([ 7.,  2.])+    >>> assert not np.all(a == b)++    The different types of interpolation can be visualized graphically:++    .. plot::++        import matplotlib.pyplot as plt++        a = np.arange(4)+        p = np.linspace(0, 100, 6001)+        ax = plt.gca()+        lines = [+            ('linear', None),+            ('higher', '--'),+            ('lower', '--'),+            ('nearest', '-.'),+            ('midpoint', '-.'),+        ]+        for interpolation, style in lines:+            ax.plot(+                p, np.percentile(a, p, interpolation=interpolation),+                label=interpolation, linestyle=style)+        ax.set(+            title='Interpolation methods for list: ' + str(a),+            xlabel='Percentile',+            ylabel='List item returned',+            yticks=a)+        ax.legend()+        plt.show()++    """+    q = np.true_divide(q, 100.0)  # handles the asarray for us too+    if not _quantile_is_valid(q):+        raise ValueError("Percentiles must be in the range [0, 100]")+    return _quantile_unchecked(+        a, q, axis, out, overwrite_input, interpolation, keepdims)+++def quantile(a, q, axis=None, out=None,+             overwrite_input=False, interpolation='linear', keepdims=False):+    """+    Compute the q-th quantile of the data along the specified axis.+    ..versionadded:: 1.15.0++    Parameters+    ----------+    a : array_like+        Input array or object that can be converted to an array.+    q : array_like of float+        Quantile or sequence of quantiles to compute, which must be between+        0 and 1 inclusive.+    axis : {int, tuple of int, None}, optional+        Axis or axes along which the quantiles are computed. The+        default is to compute the quantile(s) along a flattened+        version of the array.+    out : ndarray, optional+        Alternative output array in which to place the result. It must+        have the same shape and buffer length as the expected output,+        but the type (of the output) will be cast if necessary.+    overwrite_input : bool, optional+        If True, then allow the input array `a` to be modified by intermediate+        calculations, to save memory. In this case, the contents of the input+        `a` after this function completes is undefined.+    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}+        This optional parameter specifies the interpolation method to+        use when the desired quantile lies between two data points+        ``i < j``:++            * linear: ``i + (j - i) * fraction``, where ``fraction``+              is the fractional part of the index surrounded by ``i``+              and ``j``.+            * lower: ``i``.+            * higher: ``j``.+            * nearest: ``i`` or ``j``, whichever is nearest.+            * midpoint: ``(i + j) / 2``.+    keepdims : bool, optional+        If this is set to True, the axes which are reduced are left in+        the result as dimensions with size one. With this option, the+        result will broadcast correctly against the original array `a`.++    Returns+    -------+    quantile : scalar or ndarray+        If `q` is a single quantile and `axis=None`, then the result+        is a scalar. If multiple quantiles are given, first axis of+        the result corresponds to the quantiles. The other axes are+        the axes that remain after the reduction of `a`. If the input+        contains integers or floats smaller than ``float64``, the output+        data-type is ``float64``. Otherwise, the output data-type is the+        same as that of the input. If `out` is specified, that array is+        returned instead.++    See Also+    --------+    mean+    percentile : equivalent to quantile, but with q in the range [0, 100].+    median : equivalent to ``quantile(..., 0.5)``+    nanquantile++    Notes+    -----+    Given a vector ``V`` of length ``N``, the q-th quantile of+    ``V`` is the value ``q`` of the way from the minimum to the+    maximum in a sorted copy of ``V``. The values and distances of+    the two nearest neighbors as well as the `interpolation` parameter+    will determine the quantile if the normalized ranking does not+    match the location of ``q`` exactly. This function is the same as+    the median if ``q=0.5``, the same as the minimum if ``q=0.0`` and the+    same as the maximum if ``q=1.0``.++    Examples+    --------+    >>> a = np.array([[10, 7, 4], [3, 2, 1]])+    >>> a+    array([[10,  7,  4],+           [ 3,  2,  1]])+    >>> np.quantile(a, 0.5)+    3.5+    >>> np.quantile(a, 0.5, axis=0)+    array([[ 6.5,  4.5,  2.5]])+    >>> np.quantile(a, 0.5, axis=1)+    array([ 7.,  2.])+    >>> np.quantile(a, 0.5, axis=1, keepdims=True)+    array([[ 7.],+           [ 2.]])+    >>> m = np.quantile(a, 0.5, axis=0)+    >>> out = np.zeros_like(m)+    >>> np.quantile(a, 0.5, axis=0, out=out)+    array([[ 6.5,  4.5,  2.5]])+    >>> m+    array([[ 6.5,  4.5,  2.5]])+    >>> b = a.copy()+    >>> np.quantile(b, 0.5, axis=1, overwrite_input=True)+    array([ 7.,  2.])+    >>> assert not np.all(a == b)+    """+    q = np.asanyarray(q)+    if not _quantile_is_valid(q):+        raise ValueError("Quantiles must be in the range [0, 1]")+    return _quantile_unchecked(+        a, q, axis, out, overwrite_input, interpolation, keepdims)+++def _quantile_unchecked(a, q, axis=None, out=None, overwrite_input=False,+                        interpolation='linear', keepdims=False):+    """Assumes that q is in [0, 1], and is an ndarray"""+    r, k = _ureduce(a, func=_quantile_ureduce_func, q=q, axis=axis, out=out,+                    overwrite_input=overwrite_input,+                    interpolation=interpolation)+    if keepdims:+        return r.reshape(q.shape + k)+    else:+        return r+++def _quantile_is_valid(q):+    # avoid expensive reductions, relevant for arrays with < O(1000) elements+    if q.ndim == 1 and q.size < 10:+        for i in range(q.size):+            if q[i] < 0.0 or q[i] > 1.0:+                return False+    else:+        # faster than any()+        if np.count_nonzero(q < 0.0) or np.count_nonzero(q > 1.0):+            return False+    return True+++def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False,+                           interpolation='linear', keepdims=False):+    a = asarray(a)+    if q.ndim == 0:+        # Do not allow 0-d arrays because following code fails for scalar+        zerod = True+        q = q[None]+    else:+        zerod = False++    # prepare a for partitioning+    if overwrite_input:+        if axis is None:+            ap = a.ravel()+        else:+            ap = a+    else:+        if axis is None:+            ap = a.flatten()+        else:+            ap = a.copy()++    if axis is None:+        axis = 0++    Nx = ap.shape[axis]+    indices = q * (Nx - 1)++    # round fractional indices according to interpolation method+    if interpolation == 'lower':+        indices = floor(indices).astype(intp)+    elif interpolation == 'higher':+        indices = ceil(indices).astype(intp)+    elif interpolation == 'midpoint':+        indices = 0.5 * (floor(indices) + ceil(indices))+    elif interpolation == 'nearest':+        indices = around(indices).astype(intp)+    elif interpolation == 'linear':+        pass  # keep index as fraction and interpolate+    else:+        raise ValueError(+            "interpolation can only be 'linear', 'lower' 'higher', "+            "'midpoint', or 'nearest'")++    n = np.array(False, dtype=bool) # check for nan's flag+    if indices.dtype == intp:  # take the points along axis+        # Check if the array contains any nan's+        if np.issubdtype(a.dtype, np.inexact):+            indices = concatenate((indices, [-1]))++        ap.partition(indices, axis=axis)+        # ensure axis with q-th is first+        ap = np.moveaxis(ap, axis, 0)+        axis = 0++        # Check if the array contains any nan's+        if np.issubdtype(a.dtype, np.inexact):+            indices = indices[:-1]+            n = np.isnan(ap[-1:, ...])++        if zerod:+            indices = indices[0]+        r = take(ap, indices, axis=axis, out=out)+++    else:  # weight the points above and below the indices+        indices_below = floor(indices).astype(intp)+        indices_above = indices_below + 1+        indices_above[indices_above > Nx - 1] = Nx - 1++        # Check if the array contains any nan's+        if np.issubdtype(a.dtype, np.inexact):+            indices_above = concatenate((indices_above, [-1]))++        weights_above = indices - indices_below+        weights_below = 1.0 - weights_above++        weights_shape = [1, ] * ap.ndim+        weights_shape[axis] = len(indices)+        weights_below.shape = weights_shape+        weights_above.shape = weights_shape++        ap.partition(concatenate((indices_below, indices_above)), axis=axis)++        # ensure axis with q-th is first+        ap = np.moveaxis(ap, axis, 0)+        weights_below = np.moveaxis(weights_below, axis, 0)+        weights_above = np.moveaxis(weights_above, axis, 0)+        axis = 0++        # Check if the array contains any nan's+        if np.issubdtype(a.dtype, np.inexact):+            indices_above = indices_above[:-1]+            n = np.isnan(ap[-1:, ...])++        x1 = take(ap, indices_below, axis=axis) * weights_below+        x2 = take(ap, indices_above, axis=axis) * weights_above++        # ensure axis with q-th is first+        x1 = np.moveaxis(x1, axis, 0)+        x2 = np.moveaxis(x2, axis, 0)++        if zerod:+            x1 = x1.squeeze(0)+            x2 = x2.squeeze(0)++        if out is not None:+            r = add(x1, x2, out=out)+        else:+            r = add(x1, x2)++    if np.any(n):+        warnings.warn("Invalid value encountered in percentile",+                      RuntimeWarning, stacklevel=3)+        if zerod:+            if ap.ndim == 1:+                if out is not None:+                    out[...] = a.dtype.type(np.nan)+                    r = out+                else:+                    r = a.dtype.type(np.nan)+            else:+                r[..., n.squeeze(0)] = a.dtype.type(np.nan)+        else:+            if r.ndim == 1:+                r[:] = a.dtype.type(np.nan)+            else:+                r[..., n.repeat(q.size, 0)] = a.dtype.type(np.nan)++    return r+++def trapz(y, x=None, dx=1.0, axis=-1):+    """+    Integrate along the given axis using the composite trapezoidal rule.++    Integrate `y` (`x`) along given axis.++    Parameters+    ----------+    y : array_like+        Input array to integrate.+    x : array_like, optional+        The sample points corresponding to the `y` values. If `x` is None,+        the sample points are assumed to be evenly spaced `dx` apart. The+        default is None.+    dx : scalar, optional+        The spacing between sample points when `x` is None. The default is 1.+    axis : int, optional+        The axis along which to integrate.++    Returns+    -------+    trapz : float+        Definite integral as approximated by trapezoidal rule.++    See Also+    --------+    sum, cumsum++    Notes+    -----+    Image [2]_ illustrates trapezoidal rule -- y-axis locations of points+    will be taken from `y` array, by default x-axis distances between+    points will be 1.0, alternatively they can be provided with `x` array+    or with `dx` scalar.  Return value will be equal to combined area under+    the red lines.+++    References+    ----------+    .. [1] Wikipedia page: https://en.wikipedia.org/wiki/Trapezoidal_rule++    .. [2] Illustration image:+           https://en.wikipedia.org/wiki/File:Composite_trapezoidal_rule_illustration.png++    Examples+    --------+    >>> np.trapz([1,2,3])+    4.0+    >>> np.trapz([1,2,3], x=[4,6,8])+    8.0+    >>> np.trapz([1,2,3], dx=2)+    8.0+    >>> a = np.arange(6).reshape(2, 3)+    >>> a+    array([[0, 1, 2],+           [3, 4, 5]])+    >>> np.trapz(a, axis=0)+    array([ 1.5,  2.5,  3.5])+    >>> np.trapz(a, axis=1)+    array([ 2.,  8.])++    """+    y = asanyarray(y)+    if x is None:+        d = dx+    else:+        x = asanyarray(x)+        if x.ndim == 1:+            d = diff(x)+            # reshape to correct shape+            shape = [1]*y.ndim+            shape[axis] = d.shape[0]+            d = d.reshape(shape)+        else:+            d = diff(x, axis=axis)+    nd = y.ndim+    slice1 = [slice(None)]*nd+    slice2 = [slice(None)]*nd+    slice1[axis] = slice(1, None)+    slice2[axis] = slice(None, -1)+    try:+        ret = (d * (y[tuple(slice1)] + y[tuple(slice2)]) / 2.0).sum(axis)+    except ValueError:+        # Operations didn't work, cast to ndarray+        d = np.asarray(d)+        y = np.asarray(y)+        ret = add.reduce(d * (y[tuple(slice1)]+y[tuple(slice2)])/2.0, axis)+    return ret+++# Based on scitools meshgrid+def meshgrid(*xi, **kwargs):+    """+    Return coordinate matrices from coordinate vectors.++    Make N-D coordinate arrays for vectorized evaluations of+    N-D scalar/vector fields over N-D grids, given+    one-dimensional coordinate arrays x1, x2,..., xn.++    .. versionchanged:: 1.9+       1-D and 0-D cases are allowed.++    Parameters+    ----------+    x1, x2,..., xn : array_like+        1-D arrays representing the coordinates of a grid.+    indexing : {'xy', 'ij'}, optional+        Cartesian ('xy', default) or matrix ('ij') indexing of output.+        See Notes for more details.++        .. versionadded:: 1.7.0+    sparse : bool, optional+        If True a sparse grid is returned in order to conserve memory.+        Default is False.++        .. versionadded:: 1.7.0+    copy : bool, optional+        If False, a view into the original arrays are returned in order to+        conserve memory.  Default is True.  Please note that+        ``sparse=False, copy=False`` will likely return non-contiguous+        arrays.  Furthermore, more than one element of a broadcast array+        may refer to a single memory location.  If you need to write to the+        arrays, make copies first.++        .. versionadded:: 1.7.0++    Returns+    -------+    X1, X2,..., XN : ndarray+        For vectors `x1`, `x2`,..., 'xn' with lengths ``Ni=len(xi)`` ,+        return ``(N1, N2, N3,...Nn)`` shaped arrays if indexing='ij'+        or ``(N2, N1, N3,...Nn)`` shaped arrays if indexing='xy'+        with the elements of `xi` repeated to fill the matrix along+        the first dimension for `x1`, the second for `x2` and so on.++    Notes+    -----+    This function supports both indexing conventions through the indexing+    keyword argument.  Giving the string 'ij' returns a meshgrid with+    matrix indexing, while 'xy' returns a meshgrid with Cartesian indexing.+    In the 2-D case with inputs of length M and N, the outputs are of shape+    (N, M) for 'xy' indexing and (M, N) for 'ij' indexing.  In the 3-D case+    with inputs of length M, N and P, outputs are of shape (N, M, P) for+    'xy' indexing and (M, N, P) for 'ij' indexing.  The difference is+    illustrated by the following code snippet::++        xv, yv = np.meshgrid(x, y, sparse=False, indexing='ij')+        for i in range(nx):+            for j in range(ny):+                # treat xv[i,j], yv[i,j]++        xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy')+        for i in range(nx):+            for j in range(ny):+                # treat xv[j,i], yv[j,i]++    In the 1-D and 0-D case, the indexing and sparse keywords have no effect.++    See Also+    --------+    index_tricks.mgrid : Construct a multi-dimensional "meshgrid"+                     using indexing notation.+    index_tricks.ogrid : Construct an open multi-dimensional "meshgrid"+                     using indexing notation.++    Examples+    --------+    >>> nx, ny = (3, 2)+    >>> x = np.linspace(0, 1, nx)+    >>> y = np.linspace(0, 1, ny)+    >>> xv, yv = np.meshgrid(x, y)+    >>> xv+    array([[ 0. ,  0.5,  1. ],+           [ 0. ,  0.5,  1. ]])+    >>> yv+    array([[ 0.,  0.,  0.],+           [ 1.,  1.,  1.]])+    >>> xv, yv = np.meshgrid(x, y, sparse=True)  # make sparse output arrays+    >>> xv+    array([[ 0. ,  0.5,  1. ]])+    >>> yv+    array([[ 0.],+           [ 1.]])++    `meshgrid` is very useful to evaluate functions on a grid.++    >>> import matplotlib.pyplot as plt+    >>> x = np.arange(-5, 5, 0.1)+    >>> y = np.arange(-5, 5, 0.1)+    >>> xx, yy = np.meshgrid(x, y, sparse=True)+    >>> z = np.sin(xx**2 + yy**2) / (xx**2 + yy**2)+    >>> h = plt.contourf(x,y,z)+    >>> plt.show()++    """+    ndim = len(xi)++    copy_ = kwargs.pop('copy', True)+    sparse = kwargs.pop('sparse', False)+    indexing = kwargs.pop('indexing', 'xy')++    if kwargs:+        raise TypeError("meshgrid() got an unexpected keyword argument '%s'"+                        % (list(kwargs)[0],))++    if indexing not in ['xy', 'ij']:+        raise ValueError(+            "Valid values for `indexing` are 'xy' and 'ij'.")++    s0 = (1,) * ndim+    output = [np.asanyarray(x).reshape(s0[:i] + (-1,) + s0[i + 1:])+              for i, x in enumerate(xi)]++    if indexing == 'xy' and ndim > 1:+        # switch first and second axis+        output[0].shape = (1, -1) + s0[2:]+        output[1].shape = (-1, 1) + s0[2:]++    if not sparse:+        # Return the full N-D matrix (not only the 1-D vector)+        output = np.broadcast_arrays(*output, subok=True)++    if copy_:+        output = [x.copy() for x in output]++    return output+++def delete(arr, obj, axis=None):+    """+    Return a new array with sub-arrays along an axis deleted. For a one+    dimensional array, this returns those entries not returned by+    `arr[obj]`.++    Parameters+    ----------+    arr : array_like+      Input array.+    obj : slice, int or array of ints+      Indicate which sub-arrays to remove.+    axis : int, optional+      The axis along which to delete the subarray defined by `obj`.+      If `axis` is None, `obj` is applied to the flattened array.++    Returns+    -------+    out : ndarray+        A copy of `arr` with the elements specified by `obj` removed. Note+        that `delete` does not occur in-place. If `axis` is None, `out` is+        a flattened array.++    See Also+    --------+    insert : Insert elements into an array.+    append : Append elements at the end of an array.++    Notes+    -----+    Often it is preferable to use a boolean mask. For example:++    >>> mask = np.ones(len(arr), dtype=bool)+    >>> mask[[0,2,4]] = False+    >>> result = arr[mask,...]++    Is equivalent to `np.delete(arr, [0,2,4], axis=0)`, but allows further+    use of `mask`.++    Examples+    --------+    >>> arr = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])+    >>> arr+    array([[ 1,  2,  3,  4],+           [ 5,  6,  7,  8],+           [ 9, 10, 11, 12]])+    >>> np.delete(arr, 1, 0)+    array([[ 1,  2,  3,  4],+           [ 9, 10, 11, 12]])++    >>> np.delete(arr, np.s_[::2], 1)+    array([[ 2,  4],+           [ 6,  8],+           [10, 12]])+    >>> np.delete(arr, [1,3,5], None)+    array([ 1,  3,  5,  7,  8,  9, 10, 11, 12])++    """+    wrap = None+    if type(arr) is not ndarray:+        try:+            wrap = arr.__array_wrap__+        except AttributeError:+            pass++    arr = asarray(arr)+    ndim = arr.ndim+    arrorder = 'F' if arr.flags.fnc else 'C'+    if axis is None:+        if ndim != 1:+            arr = arr.ravel()+        ndim = arr.ndim+        axis = -1++    if ndim == 0:+        # 2013-09-24, 1.9+        warnings.warn(+            "in the future the special handling of scalars will be removed "+            "from delete and raise an error", DeprecationWarning, stacklevel=2)+        if wrap:+            return wrap(arr)+        else:+            return arr.copy(order=arrorder)++    axis = normalize_axis_index(axis, ndim)++    slobj = [slice(None)]*ndim+    N = arr.shape[axis]+    newshape = list(arr.shape)++    if isinstance(obj, slice):+        start, stop, step = obj.indices(N)+        xr = range(start, stop, step)+        numtodel = len(xr)++        if numtodel <= 0:+            if wrap:+                return wrap(arr.copy(order=arrorder))+            else:+                return arr.copy(order=arrorder)++        # Invert if step is negative:+        if step < 0:+            step = -step+            start = xr[-1]+            stop = xr[0] + 1++        newshape[axis] -= numtodel+        new = empty(newshape, arr.dtype, arrorder)+        # copy initial chunk+        if start == 0:+            pass+        else:+            slobj[axis] = slice(None, start)+            new[tuple(slobj)] = arr[tuple(slobj)]+        # copy end chunck+        if stop == N:+            pass+        else:+            slobj[axis] = slice(stop-numtodel, None)+            slobj2 = [slice(None)]*ndim+            slobj2[axis] = slice(stop, None)+            new[tuple(slobj)] = arr[tuple(slobj2)]+        # copy middle pieces+        if step == 1:+            pass+        else:  # use array indexing.+            keep = ones(stop-start, dtype=bool)+            keep[:stop-start:step] = False+            slobj[axis] = slice(start, stop-numtodel)+            slobj2 = [slice(None)]*ndim+            slobj2[axis] = slice(start, stop)+            arr = arr[tuple(slobj2)]+            slobj2[axis] = keep+            new[tuple(slobj)] = arr[tuple(slobj2)]+        if wrap:+            return wrap(new)+        else:+            return new++    _obj = obj+    obj = np.asarray(obj)+    # After removing the special handling of booleans and out of+    # bounds values, the conversion to the array can be removed.+    if obj.dtype == bool:+        warnings.warn("in the future insert will treat boolean arrays and "+                      "array-likes as boolean index instead of casting it "+                      "to integer", FutureWarning, stacklevel=2)+        obj = obj.astype(intp)+    if isinstance(_obj, (int, long, integer)):+        # optimization for a single value+        obj = obj.item()+        if (obj < -N or obj >= N):+            raise IndexError(+                "index %i is out of bounds for axis %i with "+                "size %i" % (obj, axis, N))+        if (obj < 0):+            obj += N+        newshape[axis] -= 1+        new = empty(newshape, arr.dtype, arrorder)+        slobj[axis] = slice(None, obj)+        new[tuple(slobj)] = arr[tuple(slobj)]+        slobj[axis] = slice(obj, None)+        slobj2 = [slice(None)]*ndim+        slobj2[axis] = slice(obj+1, None)+        new[tuple(slobj)] = arr[tuple(slobj2)]+    else:+        if obj.size == 0 and not isinstance(_obj, np.ndarray):+            obj = obj.astype(intp)+        if not np.can_cast(obj, intp, 'same_kind'):+            # obj.size = 1 special case always failed and would just+            # give superfluous warnings.+            # 2013-09-24, 1.9+            warnings.warn(+                "using a non-integer array as obj in delete will result in an "+                "error in the future", DeprecationWarning, stacklevel=2)+            obj = obj.astype(intp)+        keep = ones(N, dtype=bool)++        # Test if there are out of bound indices, this is deprecated+        inside_bounds = (obj < N) & (obj >= -N)+        if not inside_bounds.all():+            # 2013-09-24, 1.9+            warnings.warn(+                "in the future out of bounds indices will raise an error "+                "instead of being ignored by `numpy.delete`.",+                DeprecationWarning, stacklevel=2)+            obj = obj[inside_bounds]+        positive_indices = obj >= 0+        if not positive_indices.all():+            warnings.warn(+                "in the future negative indices will not be ignored by "+                "`numpy.delete`.", FutureWarning, stacklevel=2)+            obj = obj[positive_indices]++        keep[obj, ] = False+        slobj[axis] = keep+        new = arr[tuple(slobj)]++    if wrap:+        return wrap(new)+    else:+        return new+++def insert(arr, obj, values, axis=None):+    """+    Insert values along the given axis before the given indices.++    Parameters+    ----------+    arr : array_like+        Input array.+    obj : int, slice or sequence of ints+        Object that defines the index or indices before which `values` is+        inserted.++        .. versionadded:: 1.8.0++        Support for multiple insertions when `obj` is a single scalar or a+        sequence with one element (similar to calling insert multiple+        times).+    values : array_like+        Values to insert into `arr`. If the type of `values` is different+        from that of `arr`, `values` is converted to the type of `arr`.+        `values` should be shaped so that ``arr[...,obj,...] = values``+        is legal.+    axis : int, optional+        Axis along which to insert `values`.  If `axis` is None then `arr`+        is flattened first.++    Returns+    -------+    out : ndarray+        A copy of `arr` with `values` inserted.  Note that `insert`+        does not occur in-place: a new array is returned. If+        `axis` is None, `out` is a flattened array.++    See Also+    --------+    append : Append elements at the end of an array.+    concatenate : Join a sequence of arrays along an existing axis.+    delete : Delete elements from an array.++    Notes+    -----+    Note that for higher dimensional inserts `obj=0` behaves very different+    from `obj=[0]` just like `arr[:,0,:] = values` is different from+    `arr[:,[0],:] = values`.++    Examples+    --------+    >>> a = np.array([[1, 1], [2, 2], [3, 3]])+    >>> a+    array([[1, 1],+           [2, 2],+           [3, 3]])+    >>> np.insert(a, 1, 5)+    array([1, 5, 1, 2, 2, 3, 3])+    >>> np.insert(a, 1, 5, axis=1)+    array([[1, 5, 1],+           [2, 5, 2],+           [3, 5, 3]])++    Difference between sequence and scalars:++    >>> np.insert(a, [1], [[1],[2],[3]], axis=1)+    array([[1, 1, 1],+           [2, 2, 2],+           [3, 3, 3]])+    >>> np.array_equal(np.insert(a, 1, [1, 2, 3], axis=1),+    ...                np.insert(a, [1], [[1],[2],[3]], axis=1))+    True++    >>> b = a.flatten()+    >>> b+    array([1, 1, 2, 2, 3, 3])+    >>> np.insert(b, [2, 2], [5, 6])+    array([1, 1, 5, 6, 2, 2, 3, 3])++    >>> np.insert(b, slice(2, 4), [5, 6])+    array([1, 1, 5, 2, 6, 2, 3, 3])++    >>> np.insert(b, [2, 2], [7.13, False]) # type casting+    array([1, 1, 7, 0, 2, 2, 3, 3])++    >>> x = np.arange(8).reshape(2, 4)+    >>> idx = (1, 3)+    >>> np.insert(x, idx, 999, axis=1)+    array([[  0, 999,   1,   2, 999,   3],+           [  4, 999,   5,   6, 999,   7]])++    """+    wrap = None+    if type(arr) is not ndarray:+        try:+            wrap = arr.__array_wrap__+        except AttributeError:+            pass++    arr = asarray(arr)+    ndim = arr.ndim+    arrorder = 'F' if arr.flags.fnc else 'C'+    if axis is None:+        if ndim != 1:+            arr = arr.ravel()+        ndim = arr.ndim+        axis = ndim - 1+    elif ndim == 0:+        # 2013-09-24, 1.9+        warnings.warn(+            "in the future the special handling of scalars will be removed "+            "from insert and raise an error", DeprecationWarning, stacklevel=2)+        arr = arr.copy(order=arrorder)+        arr[...] = values+        if wrap:+            return wrap(arr)+        else:+            return arr+    else:+        axis = normalize_axis_index(axis, ndim)+    slobj = [slice(None)]*ndim+    N = arr.shape[axis]+    newshape = list(arr.shape)++    if isinstance(obj, slice):+        # turn it into a range object+        indices = arange(*obj.indices(N), **{'dtype': intp})+    else:+        # need to copy obj, because indices will be changed in-place+        indices = np.array(obj)+        if indices.dtype == bool:+            # See also delete+            warnings.warn(+                "in the future insert will treat boolean arrays and "+                "array-likes as a boolean index instead of casting it to "+                "integer", FutureWarning, stacklevel=2)+            indices = indices.astype(intp)+            # Code after warning period:+            #if obj.ndim != 1:+            #    raise ValueError('boolean array argument obj to insert '+            #                     'must be one dimensional')+            #indices = np.flatnonzero(obj)+        elif indices.ndim > 1:+            raise ValueError(+                "index array argument obj to insert must be one dimensional "+                "or scalar")+    if indices.size == 1:+        index = indices.item()+        if index < -N or index > N:+            raise IndexError(+                "index %i is out of bounds for axis %i with "+                "size %i" % (obj, axis, N))+        if (index < 0):+            index += N++        # There are some object array corner cases here, but we cannot avoid+        # that:+        values = array(values, copy=False, ndmin=arr.ndim, dtype=arr.dtype)+        if indices.ndim == 0:+            # broadcasting is very different here, since a[:,0,:] = ... behaves+            # very different from a[:,[0],:] = ...! This changes values so that+            # it works likes the second case. (here a[:,0:1,:])+            values = np.moveaxis(values, 0, axis)+        numnew = values.shape[axis]+        newshape[axis] += numnew+        new = empty(newshape, arr.dtype, arrorder)+        slobj[axis] = slice(None, index)+        new[tuple(slobj)] = arr[tuple(slobj)]+        slobj[axis] = slice(index, index+numnew)+        new[tuple(slobj)] = values+        slobj[axis] = slice(index+numnew, None)+        slobj2 = [slice(None)] * ndim+        slobj2[axis] = slice(index, None)+        new[tuple(slobj)] = arr[tuple(slobj2)]+        if wrap:+            return wrap(new)+        return new+    elif indices.size == 0 and not isinstance(obj, np.ndarray):+        # Can safely cast the empty list to intp+        indices = indices.astype(intp)++    if not np.can_cast(indices, intp, 'same_kind'):+        # 2013-09-24, 1.9+        warnings.warn(+            "using a non-integer array as obj in insert will result in an "+            "error in the future", DeprecationWarning, stacklevel=2)+        indices = indices.astype(intp)++    indices[indices < 0] += N++    numnew = len(indices)+    order = indices.argsort(kind='mergesort')   # stable sort+    indices[order] += np.arange(numnew)++    newshape[axis] += numnew+    old_mask = ones(newshape[axis], dtype=bool)+    old_mask[indices] = False++    new = empty(newshape, arr.dtype, arrorder)+    slobj2 = [slice(None)]*ndim+    slobj[axis] = indices+    slobj2[axis] = old_mask+    new[tuple(slobj)] = values+    new[tuple(slobj2)] = arr++    if wrap:+        return wrap(new)+    return new+++def append(arr, values, axis=None):+    """+    Append values to the end of an array.++    Parameters+    ----------+    arr : array_like+        Values are appended to a copy of this array.+    values : array_like+        These values are appended to a copy of `arr`.  It must be of the+        correct shape (the same shape as `arr`, excluding `axis`).  If+        `axis` is not specified, `values` can be any shape and will be+        flattened before use.+    axis : int, optional+        The axis along which `values` are appended.  If `axis` is not+        given, both `arr` and `values` are flattened before use.++    Returns+    -------+    append : ndarray+        A copy of `arr` with `values` appended to `axis`.  Note that+        `append` does not occur in-place: a new array is allocated and+        filled.  If `axis` is None, `out` is a flattened array.++    See Also+    --------+    insert : Insert elements into an array.+    delete : Delete elements from an array.++    Examples+    --------+    >>> np.append([1, 2, 3], [[4, 5, 6], [7, 8, 9]])+    array([1, 2, 3, 4, 5, 6, 7, 8, 9])++    When `axis` is specified, `values` must have the correct shape.++    >>> np.append([[1, 2, 3], [4, 5, 6]], [[7, 8, 9]], axis=0)+    array([[1, 2, 3],+           [4, 5, 6],+           [7, 8, 9]])+    >>> np.append([[1, 2, 3], [4, 5, 6]], [7, 8, 9], axis=0)+    Traceback (most recent call last):+    ...+    ValueError: arrays must have same number of dimensions++    """+    arr = asanyarray(arr)+    if axis is None:+        if arr.ndim != 1:+            arr = arr.ravel()+        values = ravel(values)+        axis = arr.ndim-1+    return concatenate((arr, values), axis=axis)+++def digitize(x, bins, right=False):+    """+    Return the indices of the bins to which each value in input array belongs.++    =========  =============  ============================+    `right`    order of bins  returned index `i` satisfies+    =========  =============  ============================+    ``False``  increasing     ``bins[i-1] <= x < bins[i]``+    ``True``   increasing     ``bins[i-1] < x <= bins[i]``+    ``False``  decreasing     ``bins[i-1] > x >= bins[i]``+    ``True``   decreasing     ``bins[i-1] >= x > bins[i]``+    =========  =============  ============================++    If values in `x` are beyond the bounds of `bins`, 0 or ``len(bins)`` is+    returned as appropriate.++    Parameters+    ----------+    x : array_like+        Input array to be binned. Prior to NumPy 1.10.0, this array had to+        be 1-dimensional, but can now have any shape.+    bins : array_like+        Array of bins. It has to be 1-dimensional and monotonic.+    right : bool, optional+        Indicating whether the intervals include the right or the left bin+        edge. Default behavior is (right==False) indicating that the interval+        does not include the right edge. The left bin end is open in this+        case, i.e., bins[i-1] <= x < bins[i] is the default behavior for+        monotonically increasing bins.++    Returns+    -------+    indices : ndarray of ints+        Output array of indices, of same shape as `x`.++    Raises+    ------+    ValueError+        If `bins` is not monotonic.+    TypeError+        If the type of the input is complex.++    See Also+    --------+    bincount, histogram, unique, searchsorted++    Notes+    -----+    If values in `x` are such that they fall outside the bin range,+    attempting to index `bins` with the indices that `digitize` returns+    will result in an IndexError.++    .. versionadded:: 1.10.0++    `np.digitize` is  implemented in terms of `np.searchsorted`. This means+    that a binary search is used to bin the values, which scales much better+    for larger number of bins than the previous linear search. It also removes+    the requirement for the input array to be 1-dimensional.++    For monotonically _increasing_ `bins`, the following are equivalent::++        np.digitize(x, bins, right=True)+        np.searchsorted(bins, x, side='left')++    Note that as the order of the arguments are reversed, the side must be too.+    The `searchsorted` call is marginally faster, as it does not do any+    monotonicity checks. Perhaps more importantly, it supports all dtypes.++    Examples+    --------+    >>> x = np.array([0.2, 6.4, 3.0, 1.6])+    >>> bins = np.array([0.0, 1.0, 2.5, 4.0, 10.0])+    >>> inds = np.digitize(x, bins)+    >>> inds+    array([1, 4, 3, 2])+    >>> for n in range(x.size):+    ...   print(bins[inds[n]-1], "<=", x[n], "<", bins[inds[n]])+    ...+    0.0 <= 0.2 < 1.0+    4.0 <= 6.4 < 10.0+    2.5 <= 3.0 < 4.0+    1.0 <= 1.6 < 2.5++    >>> x = np.array([1.2, 10.0, 12.4, 15.5, 20.])+    >>> bins = np.array([0, 5, 10, 15, 20])+    >>> np.digitize(x,bins,right=True)+    array([1, 2, 3, 4, 4])+    >>> np.digitize(x,bins,right=False)+    array([1, 3, 3, 4, 5])+    """+    x = _nx.asarray(x)+    bins = _nx.asarray(bins)++    # here for compatibility, searchsorted below is happy to take this+    if np.issubdtype(x.dtype, _nx.complexfloating):+        raise TypeError("x may not be complex")++    mono = _monotonicity(bins)+    if mono == 0:+        raise ValueError("bins must be monotonically increasing or decreasing")++    # this is backwards because the arguments below are swapped+    side = 'left' if right else 'right'+    if mono == -1:+        # reverse the bins, and invert the results+        return len(bins) - _nx.searchsorted(bins[::-1], x, side=side)+    else:+        return _nx.searchsorted(bins, x, side=side)
+ test/files/pandas.py view
@@ -0,0 +1,10182 @@+# pylint: disable=W0231,E1101+import collections+import functools+import warnings+import operator+import weakref+import gc+import json++import numpy as np+import pandas as pd++from pandas._libs import tslib, properties+from pandas.core.dtypes.common import (+    ensure_int64,+    ensure_object,+    is_scalar,+    is_number,+    is_integer, is_bool,+    is_bool_dtype,+    is_categorical_dtype,+    is_numeric_dtype,+    is_datetime64_any_dtype,+    is_timedelta64_dtype,+    is_datetime64tz_dtype,+    is_list_like,+    is_dict_like,+    is_re_compilable,+    is_period_arraylike,+    is_object_dtype,+    pandas_dtype)+from pandas.core.dtypes.cast import maybe_promote, maybe_upcast_putmask+from pandas.core.dtypes.inference import is_hashable+from pandas.core.dtypes.missing import isna, notna+from pandas.core.dtypes.generic import ABCSeries, ABCPanel, ABCDataFrame++from pandas.core.base import PandasObject, SelectionMixin+from pandas.core.index import (Index, MultiIndex, ensure_index,+                               InvalidIndexError, RangeIndex)+import pandas.core.indexing as indexing+from pandas.core.indexes.datetimes import DatetimeIndex+from pandas.core.indexes.period import PeriodIndex, Period+from pandas.core.internals import BlockManager+import pandas.core.algorithms as algos+import pandas.core.common as com+import pandas.core.missing as missing+from pandas.io.formats.printing import pprint_thing+from pandas.io.formats.format import format_percentiles, DataFrameFormatter+from pandas.tseries.frequencies import to_offset+from pandas import compat+from pandas.compat.numpy import function as nv+from pandas.compat import (map, zip, lzip, lrange, string_types, to_str,+                           isidentifier, set_function_name, cPickle as pkl)+from pandas.core.ops import _align_method_FRAME+import pandas.core.nanops as nanops+from pandas.util._decorators import (Appender, Substitution,+                                     deprecate_kwarg)+from pandas.util._validators import validate_bool_kwarg, validate_fillna_kwargs+from pandas.core import config++# goal is to be able to define the docs close to function, while still being+# able to share+_shared_docs = dict()+_shared_doc_kwargs = dict(+    axes='keywords for axes', klass='NDFrame',+    axes_single_arg='int or labels for object',+    args_transpose='axes to permute (int or label for object)',+    optional_by="""+        by : str or list of str+            Name or list of names to sort by""")+++def _single_replace(self, to_replace, method, inplace, limit):+    """+    Replaces values in a Series using the fill method specified when no+    replacement value is given in the replace method+    """+    if self.ndim != 1:+        raise TypeError('cannot replace {0} with method {1} on a {2}'+                        .format(to_replace, method, type(self).__name__))++    orig_dtype = self.dtype+    result = self if inplace else self.copy()+    fill_f = missing.get_fill_func(method)++    mask = missing.mask_missing(result.values, to_replace)+    values = fill_f(result.values, limit=limit, mask=mask)++    if values.dtype == orig_dtype and inplace:+        return++    result = pd.Series(values, index=self.index,+                       dtype=self.dtype).__finalize__(self)++    if inplace:+        self._update_inplace(result._data)+        return++    return result+++class NDFrame(PandasObject, SelectionMixin):+    """+    N-dimensional analogue of DataFrame. Store multi-dimensional in a+    size-mutable, labeled data structure++    Parameters+    ----------+    data : BlockManager+    axes : list+    copy : boolean, default False+    """+    _internal_names = ['_data', '_cacher', '_item_cache', '_cache', '_is_copy',+                       '_subtyp', '_name', '_index', '_default_kind',+                       '_default_fill_value', '_metadata', '__array_struct__',+                       '__array_interface__']+    _internal_names_set = set(_internal_names)+    _accessors = frozenset([])+    _deprecations = frozenset(['as_blocks', 'blocks',+                               'consolidate', 'convert_objects', 'is_copy'])+    _metadata = []+    _is_copy = None++    def __init__(self, data, axes=None, copy=False, dtype=None,+                 fastpath=False):++        if not fastpath:+            if dtype is not None:+                data = data.astype(dtype)+            elif copy:+                data = data.copy()++            if axes is not None:+                for i, ax in enumerate(axes):+                    data = data.reindex_axis(ax, axis=i)++        object.__setattr__(self, '_is_copy', None)+        object.__setattr__(self, '_data', data)+        object.__setattr__(self, '_item_cache', {})++    @property+    def is_copy(self):+        warnings.warn("Attribute 'is_copy' is deprecated and will be removed "+                      "in a future version.", FutureWarning, stacklevel=2)+        return self._is_copy++    @is_copy.setter+    def is_copy(self, msg):+        warnings.warn("Attribute 'is_copy' is deprecated and will be removed "+                      "in a future version.", FutureWarning, stacklevel=2)+        self._is_copy = msg++    def _repr_data_resource_(self):+        """+        Not a real Jupyter special repr method, but we use the same+        naming convention.+        """+        if config.get_option("display.html.table_schema"):+            data = self.head(config.get_option('display.max_rows'))+            payload = json.loads(data.to_json(orient='table'),+                                 object_pairs_hook=collections.OrderedDict)+            return payload++    def _validate_dtype(self, dtype):+        """ validate the passed dtype """++        if dtype is not None:+            dtype = pandas_dtype(dtype)++            # a compound dtype+            if dtype.kind == 'V':+                raise NotImplementedError("compound dtypes are not implemented"+                                          " in the {0} constructor"+                                          .format(self.__class__.__name__))++        return dtype++    def _init_mgr(self, mgr, axes=None, dtype=None, copy=False):+        """ passed a manager and a axes dict """+        for a, axe in axes.items():+            if axe is not None:+                mgr = mgr.reindex_axis(axe,+                                       axis=self._get_block_manager_axis(a),+                                       copy=False)++        # make a copy if explicitly requested+        if copy:+            mgr = mgr.copy()+        if dtype is not None:+            # avoid further copies if we can+            if len(mgr.blocks) > 1 or mgr.blocks[0].values.dtype != dtype:+                mgr = mgr.astype(dtype=dtype)+        return mgr++    # ----------------------------------------------------------------------+    # Construction++    @property+    def _constructor(self):+        """Used when a manipulation result has the same dimensions as the+        original.+        """+        raise com.AbstractMethodError(self)++    def __unicode__(self):+        # unicode representation based upon iterating over self+        # (since, by definition, `PandasContainers` are iterable)+        prepr = '[%s]' % ','.join(map(pprint_thing, self))+        return '%s(%s)' % (self.__class__.__name__, prepr)++    def _dir_additions(self):+        """ add the string-like attributes from the info_axis.+        If info_axis is a MultiIndex, it's first level values are used.+        """+        additions = {c for c in self._info_axis.unique(level=0)[:100]+                     if isinstance(c, string_types) and isidentifier(c)}+        return super(NDFrame, self)._dir_additions().union(additions)++    @property+    def _constructor_sliced(self):+        """Used when a manipulation result has one lower dimension(s) as the+        original, such as DataFrame single columns slicing.+        """+        raise com.AbstractMethodError(self)++    @property+    def _constructor_expanddim(self):+        """Used when a manipulation result has one higher dimension as the+        original, such as Series.to_frame() and DataFrame.to_panel()+        """+        raise NotImplementedError++    # ----------------------------------------------------------------------+    # Axis++    @classmethod+    def _setup_axes(cls, axes, info_axis=None, stat_axis=None, aliases=None,+                    slicers=None, axes_are_reversed=False, build_axes=True,+                    ns=None, docs=None):+        """Provide axes setup for the major PandasObjects.++        Parameters+        ----------+        axes : the names of the axes in order (lowest to highest)+        info_axis_num : the axis of the selector dimension (int)+        stat_axis_num : the number of axis for the default stats (int)+        aliases : other names for a single axis (dict)+        slicers : how axes slice to others (dict)+        axes_are_reversed : boolean whether to treat passed axes as+            reversed (DataFrame)+        build_axes : setup the axis properties (default True)+        """++        cls._AXIS_ORDERS = axes+        cls._AXIS_NUMBERS = {a: i for i, a in enumerate(axes)}+        cls._AXIS_LEN = len(axes)+        cls._AXIS_ALIASES = aliases or dict()+        cls._AXIS_IALIASES = {v: k for k, v in cls._AXIS_ALIASES.items()}+        cls._AXIS_NAMES = dict(enumerate(axes))+        cls._AXIS_SLICEMAP = slicers or None+        cls._AXIS_REVERSED = axes_are_reversed++        # typ+        setattr(cls, '_typ', cls.__name__.lower())++        # indexing support+        cls._ix = None++        if info_axis is not None:+            cls._info_axis_number = info_axis+            cls._info_axis_name = axes[info_axis]++        if stat_axis is not None:+            cls._stat_axis_number = stat_axis+            cls._stat_axis_name = axes[stat_axis]++        # setup the actual axis+        if build_axes:++            def set_axis(a, i):+                setattr(cls, a, properties.AxisProperty(i, docs.get(a, a)))+                cls._internal_names_set.add(a)++            if axes_are_reversed:+                m = cls._AXIS_LEN - 1+                for i, a in cls._AXIS_NAMES.items():+                    set_axis(a, m - i)+            else:+                for i, a in cls._AXIS_NAMES.items():+                    set_axis(a, i)++        # addtl parms+        if isinstance(ns, dict):+            for k, v in ns.items():+                setattr(cls, k, v)++    def _construct_axes_dict(self, axes=None, **kwargs):+        """Return an axes dictionary for myself."""+        d = {a: self._get_axis(a) for a in (axes or self._AXIS_ORDERS)}+        d.update(kwargs)+        return d++    @staticmethod+    def _construct_axes_dict_from(self, axes, **kwargs):+        """Return an axes dictionary for the passed axes."""+        d = {a: ax for a, ax in zip(self._AXIS_ORDERS, axes)}+        d.update(kwargs)+        return d++    def _construct_axes_dict_for_slice(self, axes=None, **kwargs):+        """Return an axes dictionary for myself."""+        d = {self._AXIS_SLICEMAP[a]: self._get_axis(a)+             for a in (axes or self._AXIS_ORDERS)}+        d.update(kwargs)+        return d++    def _construct_axes_from_arguments(self, args, kwargs, require_all=False):+        """Construct and returns axes if supplied in args/kwargs.++        If require_all, raise if all axis arguments are not supplied+        return a tuple of (axes, kwargs).+        """++        # construct the args+        args = list(args)+        for a in self._AXIS_ORDERS:++            # if we have an alias for this axis+            alias = self._AXIS_IALIASES.get(a)+            if alias is not None:+                if a in kwargs:+                    if alias in kwargs:+                        raise TypeError("arguments are mutually exclusive "+                                        "for [%s,%s]" % (a, alias))+                    continue+                if alias in kwargs:+                    kwargs[a] = kwargs.pop(alias)+                    continue++            # look for a argument by position+            if a not in kwargs:+                try:+                    kwargs[a] = args.pop(0)+                except IndexError:+                    if require_all:+                        raise TypeError("not enough/duplicate arguments "+                                        "specified!")++        axes = {a: kwargs.pop(a, None) for a in self._AXIS_ORDERS}+        return axes, kwargs++    @classmethod+    def _from_axes(cls, data, axes, **kwargs):+        # for construction from BlockManager+        if isinstance(data, BlockManager):+            return cls(data, **kwargs)+        else:+            if cls._AXIS_REVERSED:+                axes = axes[::-1]+            d = cls._construct_axes_dict_from(cls, axes, copy=False)+            d.update(kwargs)+            return cls(data, **d)++    def _get_axis_number(self, axis):+        axis = self._AXIS_ALIASES.get(axis, axis)+        if is_integer(axis):+            if axis in self._AXIS_NAMES:+                return axis+        else:+            try:+                return self._AXIS_NUMBERS[axis]+            except KeyError:+                pass+        raise ValueError('No axis named {0} for object type {1}'+                         .format(axis, type(self)))++    def _get_axis_name(self, axis):+        axis = self._AXIS_ALIASES.get(axis, axis)+        if isinstance(axis, string_types):+            if axis in self._AXIS_NUMBERS:+                return axis+        else:+            try:+                return self._AXIS_NAMES[axis]+            except KeyError:+                pass+        raise ValueError('No axis named {0} for object type {1}'+                         .format(axis, type(self)))++    def _get_axis(self, axis):+        name = self._get_axis_name(axis)+        return getattr(self, name)++    def _get_block_manager_axis(self, axis):+        """Map the axis to the block_manager axis."""+        axis = self._get_axis_number(axis)+        if self._AXIS_REVERSED:+            m = self._AXIS_LEN - 1+            return m - axis+        return axis++    def _get_axis_resolvers(self, axis):+        # index or columns+        axis_index = getattr(self, axis)+        d = dict()+        prefix = axis[0]++        for i, name in enumerate(axis_index.names):+            if name is not None:+                key = level = name+            else:+                # prefix with 'i' or 'c' depending on the input axis+                # e.g., you must do ilevel_0 for the 0th level of an unnamed+                # multiiindex+                key = '{prefix}level_{i}'.format(prefix=prefix, i=i)+                level = i++            level_values = axis_index.get_level_values(level)+            s = level_values.to_series()+            s.index = axis_index+            d[key] = s++        # put the index/columns itself in the dict+        if isinstance(axis_index, MultiIndex):+            dindex = axis_index+        else:+            dindex = axis_index.to_series()++        d[axis] = dindex+        return d++    def _get_index_resolvers(self):+        d = {}+        for axis_name in self._AXIS_ORDERS:+            d.update(self._get_axis_resolvers(axis_name))+        return d++    @property+    def _info_axis(self):+        return getattr(self, self._info_axis_name)++    @property+    def _stat_axis(self):+        return getattr(self, self._stat_axis_name)++    @property+    def shape(self):+        """Return a tuple of axis dimensions"""+        return tuple(len(self._get_axis(a)) for a in self._AXIS_ORDERS)++    @property+    def axes(self):+        """Return index label(s) of the internal NDFrame"""+        # we do it this way because if we have reversed axes, then+        # the block manager shows then reversed+        return [self._get_axis(a) for a in self._AXIS_ORDERS]++    @property+    def ndim(self):+        """+        Return an int representing the number of axes / array dimensions.++        Return 1 if Series. Otherwise return 2 if DataFrame.++        See Also+        --------+        ndarray.ndim : Number of array dimensions.++        Examples+        --------+        >>> s = pd.Series({'a': 1, 'b': 2, 'c': 3})+        >>> s.ndim+        1++        >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})+        >>> df.ndim+        2+        """+        return self._data.ndim++    @property+    def size(self):+        """+        Return an int representing the number of elements in this object.++        Return the number of rows if Series. Otherwise return the number of+        rows times number of columns if DataFrame.++        See Also+        --------+        ndarray.size : Number of elements in the array.++        Examples+        --------+        >>> s = pd.Series({'a': 1, 'b': 2, 'c': 3})+        >>> s.size+        3++        >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})+        >>> df.size+        4+        """+        return np.prod(self.shape)++    @property+    def _selected_obj(self):+        """ internal compat with SelectionMixin """+        return self++    @property+    def _obj_with_exclusions(self):+        """ internal compat with SelectionMixin """+        return self++    def _expand_axes(self, key):+        new_axes = []+        for k, ax in zip(key, self.axes):+            if k not in ax:+                if type(k) != ax.dtype.type:+                    ax = ax.astype('O')+                new_axes.append(ax.insert(len(ax), k))+            else:+                new_axes.append(ax)++        return new_axes++    def set_axis(self, labels, axis=0, inplace=None):+        """+        Assign desired index to given axis.++        Indexes for column or row labels can be changed by assigning+        a list-like or Index.++        .. versionchanged:: 0.21.0++           The signature is now `labels` and `axis`, consistent with+           the rest of pandas API. Previously, the `axis` and `labels`+           arguments were respectively the first and second positional+           arguments.++        Parameters+        ----------+        labels : list-like, Index+            The values for the new index.++        axis : {0 or 'index', 1 or 'columns'}, default 0+            The axis to update. The value 0 identifies the rows, and 1+            identifies the columns.++        inplace : boolean, default None+            Whether to return a new %(klass)s instance.++            .. warning::++               ``inplace=None`` currently falls back to to True, but in a+               future version, will default to False. Use inplace=True+               explicitly rather than relying on the default.++        Returns+        -------+        renamed : %(klass)s or None+            An object of same type as caller if inplace=False, None otherwise.++        See Also+        --------+        pandas.DataFrame.rename_axis : Alter the name of the index or columns.++        Examples+        --------+        **Series**++        >>> s = pd.Series([1, 2, 3])+        >>> s+        0    1+        1    2+        2    3+        dtype: int64++        >>> s.set_axis(['a', 'b', 'c'], axis=0, inplace=False)+        a    1+        b    2+        c    3+        dtype: int64++        The original object is not modified.++        >>> s+        0    1+        1    2+        2    3+        dtype: int64++        **DataFrame**++        >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})++        Change the row labels.++        >>> df.set_axis(['a', 'b', 'c'], axis='index', inplace=False)+           A  B+        a  1  4+        b  2  5+        c  3  6++        Change the column labels.++        >>> df.set_axis(['I', 'II'], axis='columns', inplace=False)+           I  II+        0  1   4+        1  2   5+        2  3   6++        Now, update the labels inplace.++        >>> df.set_axis(['i', 'ii'], axis='columns', inplace=True)+        >>> df+           i  ii+        0  1   4+        1  2   5+        2  3   6+        """+        if is_scalar(labels):+            warnings.warn(+                'set_axis now takes "labels" as first argument, and '+                '"axis" as named parameter. The old form, with "axis" as '+                'first parameter and \"labels\" as second, is still supported '+                'but will be deprecated in a future version of pandas.',+                FutureWarning, stacklevel=2)+            labels, axis = axis, labels++        if inplace is None:+            warnings.warn(+                'set_axis currently defaults to operating inplace.\nThis '+                'will change in a future version of pandas, use '+                'inplace=True to avoid this warning.',+                FutureWarning, stacklevel=2)+            inplace = True+        if inplace:+            setattr(self, self._get_axis_name(axis), labels)+        else:+            obj = self.copy()+            obj.set_axis(labels, axis=axis, inplace=True)+            return obj++    def _set_axis(self, axis, labels):+        self._data.set_axis(axis, labels)+        self._clear_item_cache()++    _shared_docs['transpose'] = """+        Permute the dimensions of the %(klass)s++        Parameters+        ----------+        args : %(args_transpose)s+        copy : boolean, default False+            Make a copy of the underlying data. Mixed-dtype data will+            always result in a copy++        Examples+        --------+        >>> p.transpose(2, 0, 1)+        >>> p.transpose(2, 0, 1, copy=True)++        Returns+        -------+        y : same as input+        """++    @Appender(_shared_docs['transpose'] % _shared_doc_kwargs)+    def transpose(self, *args, **kwargs):++        # construct the args+        axes, kwargs = self._construct_axes_from_arguments(args, kwargs,+                                                           require_all=True)+        axes_names = tuple(self._get_axis_name(axes[a])+                           for a in self._AXIS_ORDERS)+        axes_numbers = tuple(self._get_axis_number(axes[a])+                             for a in self._AXIS_ORDERS)++        # we must have unique axes+        if len(axes) != len(set(axes)):+            raise ValueError('Must specify %s unique axes' % self._AXIS_LEN)++        new_axes = self._construct_axes_dict_from(self, [self._get_axis(x)+                                                         for x in axes_names])+        new_values = self.values.transpose(axes_numbers)+        if kwargs.pop('copy', None) or (len(args) and args[-1]):+            new_values = new_values.copy()++        nv.validate_transpose_for_generic(self, kwargs)+        return self._constructor(new_values, **new_axes).__finalize__(self)++    def swapaxes(self, axis1, axis2, copy=True):+        """+        Interchange axes and swap values axes appropriately++        Returns+        -------+        y : same as input+        """+        i = self._get_axis_number(axis1)+        j = self._get_axis_number(axis2)++        if i == j:+            if copy:+                return self.copy()+            return self++        mapping = {i: j, j: i}++        new_axes = (self._get_axis(mapping.get(k, k))+                    for k in range(self._AXIS_LEN))+        new_values = self.values.swapaxes(i, j)+        if copy:+            new_values = new_values.copy()++        return self._constructor(new_values, *new_axes).__finalize__(self)++    def droplevel(self, level, axis=0):+        """Return DataFrame with requested index / column level(s) removed.++        .. versionadded:: 0.24.0++        Parameters+        ----------+        level : int, str, or list-like+            If a string is given, must be the name of a level+            If list-like, elements must be names or positional indexes+            of levels.++        axis : {0 or 'index', 1 or 'columns'}, default 0+++        Returns+        -------+        DataFrame.droplevel()++        Examples+        --------+        >>> df = pd.DataFrame([+        ...     [1, 2, 3, 4],+        ...     [5, 6, 7, 8],+        ...     [9, 10, 11, 12]+        ... ]).set_index([0, 1]).rename_axis(['a', 'b'])++        >>> df.columns = pd.MultiIndex.from_tuples([+        ...    ('c', 'e'), ('d', 'f')+        ... ], names=['level_1', 'level_2'])++        >>> df+        level_1   c   d+        level_2   e   f+        a b+        1 2      3   4+        5 6      7   8+        9 10    11  12++        >>> df.droplevel('a')+        level_1   c   d+        level_2   e   f+        b+        2        3   4+        6        7   8+        10      11  12++        >>> df.droplevel('level2', axis=1)+        level_1   c   d+        a b+        1 2      3   4+        5 6      7   8+        9 10    11  12++        """+        labels = self._get_axis(axis)+        new_labels = labels.droplevel(level)+        result = self.set_axis(new_labels, axis=axis, inplace=False)+        return result++    def pop(self, item):+        """+        Return item and drop from frame. Raise KeyError if not found.++        Parameters+        ----------+        item : str+            Column label to be popped++        Returns+        -------+        popped : Series++        Examples+        --------+        >>> df = pd.DataFrame([('falcon', 'bird',    389.0),+        ...                    ('parrot', 'bird',     24.0),+        ...                    ('lion',   'mammal',   80.5),+        ...                    ('monkey', 'mammal', np.nan)],+        ...                   columns=('name', 'class', 'max_speed'))+        >>> df+             name   class  max_speed+        0  falcon    bird      389.0+        1  parrot    bird       24.0+        2    lion  mammal       80.5+        3  monkey  mammal        NaN++        >>> df.pop('class')+        0      bird+        1      bird+        2    mammal+        3    mammal+        Name: class, dtype: object++        >>> df+             name  max_speed+        0  falcon      389.0+        1  parrot       24.0+        2    lion       80.5+        3  monkey        NaN+        """+        result = self[item]+        del self[item]+        try:+            result._reset_cacher()+        except AttributeError:+            pass++        return result++    def squeeze(self, axis=None):+        """+        Squeeze 1 dimensional axis objects into scalars.++        Series or DataFrames with a single element are squeezed to a scalar.+        DataFrames with a single column or a single row are squeezed to a+        Series. Otherwise the object is unchanged.++        This method is most useful when you don't know if your+        object is a Series or DataFrame, but you do know it has just a single+        column. In that case you can safely call `squeeze` to ensure you have a+        Series.++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns', None}, default None+            A specific axis to squeeze. By default, all length-1 axes are+            squeezed.++            .. versionadded:: 0.20.0++        Returns+        -------+        DataFrame, Series, or scalar+            The projection after squeezing `axis` or all the axes.++        See Also+        --------+        Series.iloc : Integer-location based indexing for selecting scalars+        DataFrame.iloc : Integer-location based indexing for selecting Series+        Series.to_frame : Inverse of DataFrame.squeeze for a+            single-column DataFrame.++        Examples+        --------+        >>> primes = pd.Series([2, 3, 5, 7])++        Slicing might produce a Series with a single value:++        >>> even_primes = primes[primes % 2 == 0]+        >>> even_primes+        0    2+        dtype: int64++        >>> even_primes.squeeze()+        2++        Squeezing objects with more than one value in every axis does nothing:++        >>> odd_primes = primes[primes % 2 == 1]+        >>> odd_primes+        1    3+        2    5+        3    7+        dtype: int64++        >>> odd_primes.squeeze()+        1    3+        2    5+        3    7+        dtype: int64++        Squeezing is even more effective when used with DataFrames.++        >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['a', 'b'])+        >>> df+           a  b+        0  1  2+        1  3  4++        Slicing a single column will produce a DataFrame with the columns+        having only one value:++        >>> df_a = df[['a']]+        >>> df_a+           a+        0  1+        1  3++        So the columns can be squeezed down, resulting in a Series:++        >>> df_a.squeeze('columns')+        0    1+        1    3+        Name: a, dtype: int64++        Slicing a single row from a single column will produce a single+        scalar DataFrame:++        >>> df_0a = df.loc[df.index < 1, ['a']]+        >>> df_0a+           a+        0  1++        Squeezing the rows produces a single scalar Series:++        >>> df_0a.squeeze('rows')+        a    1+        Name: 0, dtype: int64++        Squeezing all axes wil project directly into a scalar:++        >>> df_0a.squeeze()+        1+        """+        axis = (self._AXIS_NAMES if axis is None else+                (self._get_axis_number(axis),))+        try:+            return self.iloc[+                tuple(0 if i in axis and len(a) == 1 else slice(None)+                      for i, a in enumerate(self.axes))]+        except Exception:+            return self++    def swaplevel(self, i=-2, j=-1, axis=0):+        """+        Swap levels i and j in a MultiIndex on a particular axis++        Parameters+        ----------+        i, j : int, string (can be mixed)+            Level of index to be swapped. Can pass level name as string.++        Returns+        -------+        swapped : same type as caller (new object)++        .. versionchanged:: 0.18.1++           The indexes ``i`` and ``j`` are now optional, and default to+           the two innermost levels of the index.++        """+        axis = self._get_axis_number(axis)+        result = self.copy()+        labels = result._data.axes[axis]+        result._data.set_axis(axis, labels.swaplevel(i, j))+        return result++    # ----------------------------------------------------------------------+    # Rename++    # TODO: define separate funcs for DataFrame, Series and Panel so you can+    # get completion on keyword arguments.+    _shared_docs['rename'] = """+        Alter axes input function or functions. Function / dict values must be+        unique (1-to-1). Labels not contained in a dict / Series will be left+        as-is. Extra labels listed don't throw an error. Alternatively, change+        ``Series.name`` with a scalar value (Series only).++        Parameters+        ----------+        %(optional_mapper)s+        %(axes)s : scalar, list-like, dict-like or function, optional+            Scalar or list-like will alter the ``Series.name`` attribute,+            and raise on DataFrame or Panel.+            dict-like or functions are transformations to apply to+            that axis' values+        %(optional_axis)s+        copy : boolean, default True+            Also copy underlying data+        inplace : boolean, default False+            Whether to return a new %(klass)s. If True then value of copy is+            ignored.+        level : int or level name, default None+            In case of a MultiIndex, only rename labels in the specified+            level.++        Returns+        -------+        renamed : %(klass)s (new object)++        See Also+        --------+        pandas.NDFrame.rename_axis++        Examples+        --------++        >>> s = pd.Series([1, 2, 3])+        >>> s+        0    1+        1    2+        2    3+        dtype: int64+        >>> s.rename("my_name") # scalar, changes Series.name+        0    1+        1    2+        2    3+        Name: my_name, dtype: int64+        >>> s.rename(lambda x: x ** 2)  # function, changes labels+        0    1+        1    2+        4    3+        dtype: int64+        >>> s.rename({1: 3, 2: 5})  # mapping, changes labels+        0    1+        3    2+        5    3+        dtype: int64++        Since ``DataFrame`` doesn't have a ``.name`` attribute,+        only mapping-type arguments are allowed.++        >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})+        >>> df.rename(2)+        Traceback (most recent call last):+        ...+        TypeError: 'int' object is not callable++        ``DataFrame.rename`` supports two calling conventions++        * ``(index=index_mapper, columns=columns_mapper, ...)``+        * ``(mapper, axis={'index', 'columns'}, ...)``++        We *highly* recommend using keyword arguments to clarify your+        intent.++        >>> df.rename(index=str, columns={"A": "a", "B": "c"})+           a  c+        0  1  4+        1  2  5+        2  3  6++        >>> df.rename(index=str, columns={"A": "a", "C": "c"})+           a  B+        0  1  4+        1  2  5+        2  3  6++        Using axis-style parameters++        >>> df.rename(str.lower, axis='columns')+           a  b+        0  1  4+        1  2  5+        2  3  6++        >>> df.rename({1: 2, 2: 4}, axis='index')+           A  B+        0  1  4+        2  2  5+        4  3  6++        See the :ref:`user guide <basics.rename>` for more.+        """++    @Appender(_shared_docs['rename'] % dict(axes='axes keywords for this'+                                            ' object', klass='NDFrame',+                                            optional_mapper='',+                                            optional_axis=''))+    def rename(self, *args, **kwargs):+        axes, kwargs = self._construct_axes_from_arguments(args, kwargs)+        copy = kwargs.pop('copy', True)+        inplace = kwargs.pop('inplace', False)+        level = kwargs.pop('level', None)+        axis = kwargs.pop('axis', None)+        if axis is not None:+            # Validate the axis+            self._get_axis_number(axis)++        if kwargs:+            raise TypeError('rename() got an unexpected keyword '+                            'argument "{0}"'.format(list(kwargs.keys())[0]))++        if com.count_not_none(*axes.values()) == 0:+            raise TypeError('must pass an index to rename')++        # renamer function if passed a dict+        def _get_rename_function(mapper):+            if isinstance(mapper, (dict, ABCSeries)):++                def f(x):+                    if x in mapper:+                        return mapper[x]+                    else:+                        return x+            else:+                f = mapper++            return f++        self._consolidate_inplace()+        result = self if inplace else self.copy(deep=copy)++        # start in the axis order to eliminate too many copies+        for axis in lrange(self._AXIS_LEN):+            v = axes.get(self._AXIS_NAMES[axis])+            if v is None:+                continue+            f = _get_rename_function(v)++            baxis = self._get_block_manager_axis(axis)+            if level is not None:+                level = self.axes[axis]._get_level_number(level)+            result._data = result._data.rename_axis(f, axis=baxis, copy=copy,+                                                    level=level)+            result._clear_item_cache()++        if inplace:+            self._update_inplace(result._data)+        else:+            return result.__finalize__(self)++    rename.__doc__ = _shared_docs['rename']++    def rename_axis(self, mapper, axis=0, copy=True, inplace=False):+        """+        Alter the name of the index or columns.++        Parameters+        ----------+        mapper : scalar, list-like, optional+            Value to set as the axis name attribute.+        axis : {0 or 'index', 1 or 'columns'}, default 0+            The index or the name of the axis.+        copy : boolean, default True+            Also copy underlying data.+        inplace : boolean, default False+            Modifies the object directly, instead of creating a new Series+            or DataFrame.++        Returns+        -------+        renamed : Series, DataFrame, or None+            The same type as the caller or None if `inplace` is True.++        Notes+        -----+        Prior to version 0.21.0, ``rename_axis`` could also be used to change+        the axis *labels* by passing a mapping or scalar. This behavior is+        deprecated and will be removed in a future version. Use ``rename``+        instead.++        See Also+        --------+        pandas.Series.rename : Alter Series index labels or name+        pandas.DataFrame.rename : Alter DataFrame index labels or name+        pandas.Index.rename : Set new names on index++        Examples+        --------+        **Series**++        >>> s = pd.Series([1, 2, 3])+        >>> s.rename_axis("foo")+        foo+        0    1+        1    2+        2    3+        dtype: int64++        **DataFrame**++        >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})+        >>> df.rename_axis("foo")+             A  B+        foo+        0    1  4+        1    2  5+        2    3  6++        >>> df.rename_axis("bar", axis="columns")+        bar  A  B+        0    1  4+        1    2  5+        2    3  6+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        non_mapper = is_scalar(mapper) or (is_list_like(mapper) and not+                                           is_dict_like(mapper))+        if non_mapper:+            return self._set_axis_name(mapper, axis=axis, inplace=inplace)+        else:+            msg = ("Using 'rename_axis' to alter labels is deprecated. "+                   "Use '.rename' instead")+            warnings.warn(msg, FutureWarning, stacklevel=2)+            axis = self._get_axis_name(axis)+            d = {'copy': copy, 'inplace': inplace}+            d[axis] = mapper+            return self.rename(**d)++    def _set_axis_name(self, name, axis=0, inplace=False):+        """+        Alter the name or names of the axis.++        Parameters+        ----------+        name : str or list of str+            Name for the Index, or list of names for the MultiIndex+        axis : int or str+           0 or 'index' for the index; 1 or 'columns' for the columns+        inplace : bool+            whether to modify `self` directly or return a copy++            .. versionadded:: 0.21.0++        Returns+        -------+        renamed : same type as caller or None if inplace=True++        See Also+        --------+        pandas.DataFrame.rename+        pandas.Series.rename+        pandas.Index.rename++        Examples+        --------+        >>> df._set_axis_name("foo")+             A+        foo+        0    1+        1    2+        2    3+        >>> df.index = pd.MultiIndex.from_product([['A'], ['a', 'b', 'c']])+        >>> df._set_axis_name(["bar", "baz"])+                 A+        bar baz+        A   a    1+            b    2+            c    3+        """+        axis = self._get_axis_number(axis)+        idx = self._get_axis(axis).set_names(name)++        inplace = validate_bool_kwarg(inplace, 'inplace')+        renamed = self if inplace else self.copy()+        renamed.set_axis(idx, axis=axis, inplace=True)+        if not inplace:+            return renamed++    # ----------------------------------------------------------------------+    # Comparisons++    def _indexed_same(self, other):+        return all(self._get_axis(a).equals(other._get_axis(a))+                   for a in self._AXIS_ORDERS)++    def __neg__(self):+        values = com.values_from_object(self)+        if is_bool_dtype(values):+            arr = operator.inv(values)+        elif (is_numeric_dtype(values) or is_timedelta64_dtype(values)+                or is_object_dtype(values)):+            arr = operator.neg(values)+        else:+            raise TypeError("Unary negative expects numeric dtype, not {}"+                            .format(values.dtype))+        return self.__array_wrap__(arr)++    def __pos__(self):+        values = com.values_from_object(self)+        if (is_bool_dtype(values) or is_period_arraylike(values)):+            arr = values+        elif (is_numeric_dtype(values) or is_timedelta64_dtype(values)+                or is_object_dtype(values)):+            arr = operator.pos(values)+        else:+            raise TypeError("Unary plus expects numeric dtype, not {}"+                            .format(values.dtype))+        return self.__array_wrap__(arr)++    def __invert__(self):+        try:+            arr = operator.inv(com.values_from_object(self))+            return self.__array_wrap__(arr)+        except Exception:++            # inv fails with 0 len+            if not np.prod(self.shape):+                return self++            raise++    def equals(self, other):+        """+        Determines if two NDFrame objects contain the same elements. NaNs in+        the same location are considered equal.+        """+        if not isinstance(other, self._constructor):+            return False+        return self._data.equals(other._data)++    # -------------------------------------------------------------------------+    # Label or Level Combination Helpers+    #+    # A collection of helper methods for DataFrame/Series operations that+    # accept a combination of column/index labels and levels.  All such+    # operations should utilize/extend these methods when possible so that we+    # have consistent precedence and validation logic throughout the library.++    def _is_level_reference(self, key, axis=0):+        """+        Test whether a key is a level reference for a given axis.++        To be considered a level reference, `key` must be a string that:+          - (axis=0): Matches the name of an index level and does NOT match+            a column label.+          - (axis=1): Matches the name of a column level and does NOT match+            an index label.++        Parameters+        ----------+        key: str+            Potential level name for the given axis+        axis: int, default 0+            Axis that levels are associated with (0 for index, 1 for columns)++        Returns+        -------+        is_level: bool+        """+        axis = self._get_axis_number(axis)++        if self.ndim > 2:+            raise NotImplementedError(+                "_is_level_reference is not implemented for {type}"+                .format(type=type(self)))++        return (key is not None and+                is_hashable(key) and+                key in self.axes[axis].names and+                not self._is_label_reference(key, axis=axis))++    def _is_label_reference(self, key, axis=0):+        """+        Test whether a key is a label reference for a given axis.++        To be considered a label reference, `key` must be a string that:+          - (axis=0): Matches a column label+          - (axis=1): Matches an index label++        Parameters+        ----------+        key: str+            Potential label name+        axis: int, default 0+            Axis perpendicular to the axis that labels are associated with+            (0 means search for column labels, 1 means search for index labels)++        Returns+        -------+        is_label: bool+        """+        axis = self._get_axis_number(axis)+        other_axes = [ax for ax in range(self._AXIS_LEN) if ax != axis]++        if self.ndim > 2:+            raise NotImplementedError(+                "_is_label_reference is not implemented for {type}"+                .format(type=type(self)))++        return (key is not None and+                is_hashable(key) and+                any(key in self.axes[ax] for ax in other_axes))++    def _is_label_or_level_reference(self, key, axis=0):+        """+        Test whether a key is a label or level reference for a given axis.++        To be considered either a label or a level reference, `key` must be a+        string that:+          - (axis=0): Matches a column label or an index level+          - (axis=1): Matches an index label or a column level++        Parameters+        ----------+        key: str+            Potential label or level name+        axis: int, default 0+            Axis that levels are associated with (0 for index, 1 for columns)++        Returns+        -------+        is_label_or_level: bool+        """++        if self.ndim > 2:+            raise NotImplementedError(+                "_is_label_or_level_reference is not implemented for {type}"+                .format(type=type(self)))++        return (self._is_level_reference(key, axis=axis) or+                self._is_label_reference(key, axis=axis))++    def _check_label_or_level_ambiguity(self, key, axis=0, stacklevel=1):+        """+        Check whether `key` matches both a level of the input `axis` and a+        label of the other axis and raise a ``FutureWarning`` if this is the+        case.++        Note: This method will be altered to raise an ambiguity exception in+        a future version.++        Parameters+        ----------+        key: str or object+            label or level name+        axis: int, default 0+            Axis that levels are associated with (0 for index, 1 for columns)+        stacklevel: int, default 1+            Stack level used when a FutureWarning is raised (see below).++        Returns+        -------+        ambiguous: bool++        Raises+        ------+        FutureWarning+            if `key` is ambiguous. This will become an ambiguity error in a+            future version+        """++        axis = self._get_axis_number(axis)+        other_axes = [ax for ax in range(self._AXIS_LEN) if ax != axis]++        if self.ndim > 2:+            raise NotImplementedError(+                "_check_label_or_level_ambiguity is not implemented for {type}"+                .format(type=type(self)))++        if (key is not None and+                is_hashable(key) and+                key in self.axes[axis].names and+                any(key in self.axes[ax] for ax in other_axes)):++            # Build an informative and grammatical warning+            level_article, level_type = (('an', 'index')+                                         if axis == 0 else+                                         ('a', 'column'))++            label_article, label_type = (('a', 'column')+                                         if axis == 0 else+                                         ('an', 'index'))++            msg = ("'{key}' is both {level_article} {level_type} level and "+                   "{label_article} {label_type} label.\n"+                   "Defaulting to {label_type}, but this will raise an "+                   "ambiguity error in a future version"+                   ).format(key=key,+                            level_article=level_article,+                            level_type=level_type,+                            label_article=label_article,+                            label_type=label_type)++            warnings.warn(msg, FutureWarning, stacklevel=stacklevel + 1)+            return True+        else:+            return False++    def _get_label_or_level_values(self, key, axis=0, stacklevel=1):+        """+        Return a 1-D array of values associated with `key`, a label or level+        from the given `axis`.++        Retrieval logic:+          - (axis=0): Return column values if `key` matches a column label.+            Otherwise return index level values if `key` matches an index+            level.+          - (axis=1): Return row values if `key` matches an index label.+            Otherwise return column level values if 'key' matches a column+            level++        Parameters+        ----------+        key: str+            Label or level name.+        axis: int, default 0+            Axis that levels are associated with (0 for index, 1 for columns)+        stacklevel: int, default 1+            Stack level used when a FutureWarning is raised (see below).++        Returns+        -------+        values: np.ndarray++        Raises+        ------+        KeyError+            if `key` matches neither a label nor a level+        ValueError+            if `key` matches multiple labels+        FutureWarning+            if `key` is ambiguous. This will become an ambiguity error in a+            future version+        """++        axis = self._get_axis_number(axis)+        other_axes = [ax for ax in range(self._AXIS_LEN) if ax != axis]++        if self.ndim > 2:+            raise NotImplementedError(+                "_get_label_or_level_values is not implemented for {type}"+                .format(type=type(self)))++        if self._is_label_reference(key, axis=axis):+            self._check_label_or_level_ambiguity(key, axis=axis,+                                                 stacklevel=stacklevel + 1)+            values = self.xs(key, axis=other_axes[0])._values+        elif self._is_level_reference(key, axis=axis):+            values = self.axes[axis].get_level_values(key)._values+        else:+            raise KeyError(key)++        # Check for duplicates+        if values.ndim > 1:++            if other_axes and isinstance(+                    self._get_axis(other_axes[0]), MultiIndex):+                multi_message = ('\n'+                                 'For a multi-index, the label must be a '+                                 'tuple with elements corresponding to '+                                 'each level.')+            else:+                multi_message = ''++            label_axis_name = 'column' if axis == 0 else 'index'+            raise ValueError(("The {label_axis_name} label '{key}' "+                              "is not unique.{multi_message}")+                             .format(key=key,+                                     label_axis_name=label_axis_name,+                                     multi_message=multi_message))++        return values++    def _drop_labels_or_levels(self, keys, axis=0):+        """+        Drop labels and/or levels for the given `axis`.++        For each key in `keys`:+          - (axis=0): If key matches a column label then drop the column.+            Otherwise if key matches an index level then drop the level.+          - (axis=1): If key matches an index label then drop the row.+            Otherwise if key matches a column level then drop the level.++        Parameters+        ----------+        keys: str or list of str+            labels or levels to drop+        axis: int, default 0+            Axis that levels are associated with (0 for index, 1 for columns)++        Returns+        -------+        dropped: DataFrame++        Raises+        ------+        ValueError+            if any `keys` match neither a label nor a level+        """++        axis = self._get_axis_number(axis)++        if self.ndim > 2:+            raise NotImplementedError(+                "_drop_labels_or_levels is not implemented for {type}"+                .format(type=type(self)))++        # Validate keys+        keys = com.maybe_make_list(keys)+        invalid_keys = [k for k in keys if not+                        self._is_label_or_level_reference(k, axis=axis)]++        if invalid_keys:+            raise ValueError(("The following keys are not valid labels or "+                              "levels for axis {axis}: {invalid_keys}")+                             .format(axis=axis,+                                     invalid_keys=invalid_keys))++        # Compute levels and labels to drop+        levels_to_drop = [k for k in keys+                          if self._is_level_reference(k, axis=axis)]++        labels_to_drop = [k for k in keys+                          if not self._is_level_reference(k, axis=axis)]++        # Perform copy upfront and then use inplace operations below.+        # This ensures that we always perform exactly one copy.+        # ``copy`` and/or ``inplace`` options could be added in the future.+        dropped = self.copy()++        if axis == 0:+            # Handle dropping index levels+            if levels_to_drop:+                dropped.reset_index(levels_to_drop, drop=True, inplace=True)++            # Handle dropping columns labels+            if labels_to_drop:+                dropped.drop(labels_to_drop, axis=1, inplace=True)+        else:+            # Handle dropping column levels+            if levels_to_drop:+                if isinstance(dropped.columns, MultiIndex):+                    # Drop the specified levels from the MultiIndex+                    dropped.columns = dropped.columns.droplevel(levels_to_drop)+                else:+                    # Drop the last level of Index by replacing with+                    # a RangeIndex+                    dropped.columns = RangeIndex(dropped.columns.size)++            # Handle dropping index labels+            if labels_to_drop:+                dropped.drop(labels_to_drop, axis=0, inplace=True)++        return dropped++    # ----------------------------------------------------------------------+    # Iteration++    def __hash__(self):+        raise TypeError('{0!r} objects are mutable, thus they cannot be'+                        ' hashed'.format(self.__class__.__name__))++    def __iter__(self):+        """Iterate over infor axis"""+        return iter(self._info_axis)++    # can we get a better explanation of this?+    def keys(self):+        """Get the 'info axis' (see Indexing for more)++        This is index for Series, columns for DataFrame and major_axis for+        Panel.+        """+        return self._info_axis++    def iteritems(self):+        """Iterate over (label, values) on info axis++        This is index for Series, columns for DataFrame, major_axis for Panel,+        and so on.+        """+        for h in self._info_axis:+            yield h, self[h]++    def __len__(self):+        """Returns length of info axis"""+        return len(self._info_axis)++    def __contains__(self, key):+        """True if the key is in the info axis"""+        return key in self._info_axis++    @property+    def empty(self):+        """+        Indicator whether DataFrame is empty.++        True if DataFrame is entirely empty (no items), meaning any of the+        axes are of length 0.++        Returns+        -------+        bool+            If DataFrame is empty, return True, if not return False.++        Notes+        -----+        If DataFrame contains only NaNs, it is still not considered empty. See+        the example below.++        Examples+        --------+        An example of an actual empty DataFrame. Notice the index is empty:++        >>> df_empty = pd.DataFrame({'A' : []})+        >>> df_empty+        Empty DataFrame+        Columns: [A]+        Index: []+        >>> df_empty.empty+        True++        If we only have NaNs in our DataFrame, it is not considered empty! We+        will need to drop the NaNs to make the DataFrame empty:++        >>> df = pd.DataFrame({'A' : [np.nan]})+        >>> df+            A+        0 NaN+        >>> df.empty+        False+        >>> df.dropna().empty+        True++        See also+        --------+        pandas.Series.dropna+        pandas.DataFrame.dropna+        """+        return any(len(self._get_axis(a)) == 0 for a in self._AXIS_ORDERS)++    def __nonzero__(self):+        raise ValueError("The truth value of a {0} is ambiguous. "+                         "Use a.empty, a.bool(), a.item(), a.any() or a.all()."+                         .format(self.__class__.__name__))++    __bool__ = __nonzero__++    def bool(self):+        """Return the bool of a single element PandasObject.++        This must be a boolean scalar value, either True or False.  Raise a+        ValueError if the PandasObject does not have exactly 1 element, or that+        element is not boolean+        """+        v = self.squeeze()+        if isinstance(v, (bool, np.bool_)):+            return bool(v)+        elif is_scalar(v):+            raise ValueError("bool cannot act on a non-boolean single element "+                             "{0}".format(self.__class__.__name__))++        self.__nonzero__()++    def __abs__(self):+        return self.abs()++    def __round__(self, decimals=0):+        return self.round(decimals)++    # ----------------------------------------------------------------------+    # Array Interface++    def __array__(self, dtype=None):+        return com.values_from_object(self)++    def __array_wrap__(self, result, context=None):+        d = self._construct_axes_dict(self._AXIS_ORDERS, copy=False)+        return self._constructor(result, **d).__finalize__(self)++    # ideally we would define this to avoid the getattr checks, but+    # is slower+    # @property+    # def __array_interface__(self):+    #    """ provide numpy array interface method """+    #    values = self.values+    #    return dict(typestr=values.dtype.str,shape=values.shape,data=values)++    def to_dense(self):+        """Return dense representation of NDFrame (as opposed to sparse)"""+        # compat+        return self++    # ----------------------------------------------------------------------+    # Picklability++    def __getstate__(self):+        meta = {k: getattr(self, k, None) for k in self._metadata}+        return dict(_data=self._data, _typ=self._typ, _metadata=self._metadata,+                    **meta)++    def __setstate__(self, state):++        if isinstance(state, BlockManager):+            self._data = state+        elif isinstance(state, dict):+            typ = state.get('_typ')+            if typ is not None:++                # set in the order of internal names+                # to avoid definitional recursion+                # e.g. say fill_value needing _data to be+                # defined+                meta = set(self._internal_names + self._metadata)+                for k in list(meta):+                    if k in state:+                        v = state[k]+                        object.__setattr__(self, k, v)++                for k, v in state.items():+                    if k not in meta:+                        object.__setattr__(self, k, v)++            else:+                self._unpickle_series_compat(state)+        elif isinstance(state[0], dict):+            if len(state) == 5:+                self._unpickle_sparse_frame_compat(state)+            else:+                self._unpickle_frame_compat(state)+        elif len(state) == 4:+            self._unpickle_panel_compat(state)+        elif len(state) == 2:+            self._unpickle_series_compat(state)+        else:  # pragma: no cover+            # old pickling format, for compatibility+            self._unpickle_matrix_compat(state)++        self._item_cache = {}++    # ----------------------------------------------------------------------+    # IO++    def _repr_latex_(self):+        """+        Returns a LaTeX representation for a particular object.+        Mainly for use with nbconvert (jupyter notebook conversion to pdf).+        """+        if config.get_option('display.latex.repr'):+            return self.to_latex()+        else:+            return None++    # ----------------------------------------------------------------------+    # I/O Methods++    _shared_docs['to_excel'] = """+    Write %(klass)s to an excel sheet.++    To write a single %(klass)s to an excel .xlsx file it is only necessary to+    specify a target file name. To write to multiple sheets it is necessary to+    create an `ExcelWriter` object with a target file name, and specify a sheet+    in the file to write to. Multiple sheets may be written to by+    specifying unique `sheet_name`. With all data written to the file it is+    necessary to save the changes. Note that creating an ExcelWriter object+    with a file name that already exists will result in the contents of the+    existing file being erased.++    Parameters+    ----------+    excel_writer : string or ExcelWriter object+        File path or existing ExcelWriter.+    sheet_name : string, default 'Sheet1'+        Name of sheet which will contain DataFrame.+    na_rep : string, default ''+        Missing data representation.+    float_format : string, optional+        Format string for floating point numbers. For example+        ``float_format="%%.2f"`` will format 0.1234 to 0.12.+    columns : sequence or list of string, optional+        Columns to write.+    header : boolean or list of string, default True+        Write out the column names. If a list of strings is given it is+        assumed to be aliases for the column names.+    index : boolean, default True+        Write row names (index).+    index_label : string or sequence, optional+        Column label for index column(s) if desired. If not specified, and+        `header` and `index` are True, then the index names are used. A+        sequence should be given if the DataFrame uses MultiIndex.+    startrow : integer, default 0+        Upper left cell row to dump data frame.+    startcol : integer, default 0+        Upper left cell column to dump data frame.+    engine : string, optional+        Write engine to use, 'openpyxl' or 'xlsxwriter'. You can also set this+        via the options ``io.excel.xlsx.writer``, ``io.excel.xls.writer``, and+        ``io.excel.xlsm.writer``.+    merge_cells : boolean, default True+        Write MultiIndex and Hierarchical Rows as merged cells.+    encoding : string, optional+        Encoding of the resulting excel file. Only necessary for xlwt,+        other writers support unicode natively.+    inf_rep : string, default 'inf'+        Representation for infinity (there is no native representation for+        infinity in Excel).+    verbose : boolean, default True+        Display more information in the error logs.+    freeze_panes : tuple of integer (length 2), optional+        Specifies the one-based bottommost row and rightmost column that+        is to be frozen.++        .. versionadded:: 0.20.0.++    Notes+    -----+    For compatibility with :meth:`~DataFrame.to_csv`,+    to_excel serializes lists and dicts to strings before writing.++    Once a workbook has been saved it is not possible write further data+    without rewriting the whole workbook.++    See Also+    --------+    pandas.read_excel+    pandas.ExcelWriter++    Examples+    --------++    Create, write to and save a workbook:++    >>> df1 = pd.DataFrame([['a', 'b'], ['c', 'd']],+    ...                   index=['row 1', 'row 2'],+    ...                   columns=['col 1', 'col 2'])+    >>> df1.to_excel("output.xlsx")++    To specify the sheet name:++    >>> df1.to_excel("output.xlsx", sheet_name='Sheet_name_1')++    If you wish to write to more than one sheet in the workbook, it is+    necessary to specify an ExcelWriter object:++    >>> writer = pd.ExcelWriter('output2.xlsx', engine='xlsxwriter')+    >>> df1.to_excel(writer, sheet_name='Sheet1')+    >>> df2 = df1.copy()+    >>> df2.to_excel(writer, sheet_name='Sheet2')+    >>> writer.save()+    """++    def to_json(self, path_or_buf=None, orient=None, date_format=None,+                double_precision=10, force_ascii=True, date_unit='ms',+                default_handler=None, lines=False, compression='infer',+                index=True):+        """+        Convert the object to a JSON string.++        Note NaN's and None will be converted to null and datetime objects+        will be converted to UNIX timestamps.++        Parameters+        ----------+        path_or_buf : string or file handle, optional+            File path or object. If not specified, the result is returned as+            a string.+        orient : string+            Indication of expected JSON string format.++            * Series++              - default is 'index'+              - allowed values are: {'split','records','index','table'}++            * DataFrame++              - default is 'columns'+              - allowed values are:+                {'split','records','index','columns','values','table'}++            * The format of the JSON string++              - 'split' : dict like {'index' -> [index],+                'columns' -> [columns], 'data' -> [values]}+              - 'records' : list like+                [{column -> value}, ... , {column -> value}]+              - 'index' : dict like {index -> {column -> value}}+              - 'columns' : dict like {column -> {index -> value}}+              - 'values' : just the values array+              - 'table' : dict like {'schema': {schema}, 'data': {data}}+                describing the data, and the data component is+                like ``orient='records'``.++                .. versionchanged:: 0.20.0++        date_format : {None, 'epoch', 'iso'}+            Type of date conversion. 'epoch' = epoch milliseconds,+            'iso' = ISO8601. The default depends on the `orient`. For+            ``orient='table'``, the default is 'iso'. For all other orients,+            the default is 'epoch'.+        double_precision : int, default 10+            The number of decimal places to use when encoding+            floating point values.+        force_ascii : boolean, default True+            Force encoded string to be ASCII.+        date_unit : string, default 'ms' (milliseconds)+            The time unit to encode to, governs timestamp and ISO8601+            precision.  One of 's', 'ms', 'us', 'ns' for second, millisecond,+            microsecond, and nanosecond respectively.+        default_handler : callable, default None+            Handler to call if object cannot otherwise be converted to a+            suitable format for JSON. Should receive a single argument which is+            the object to convert and return a serialisable object.+        lines : boolean, default False+            If 'orient' is 'records' write out line delimited json format. Will+            throw ValueError if incorrect 'orient' since others are not list+            like.++            .. versionadded:: 0.19.0+        compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None},+                       default 'infer'+            A string representing the compression to use in the output file,+            only used when the first argument is a filename.++            .. versionadded:: 0.21.0+            .. versionchanged:: 0.24.0+               'infer' option added and set to default+        index : boolean, default True+            Whether to include the index values in the JSON string. Not+            including the index (``index=False``) is only supported when+            orient is 'split' or 'table'.++            .. versionadded:: 0.23.0++        See Also+        --------+        pandas.read_json++        Examples+        --------++        >>> df = pd.DataFrame([['a', 'b'], ['c', 'd']],+        ...                   index=['row 1', 'row 2'],+        ...                   columns=['col 1', 'col 2'])+        >>> df.to_json(orient='split')+        '{"columns":["col 1","col 2"],+          "index":["row 1","row 2"],+          "data":[["a","b"],["c","d"]]}'++        Encoding/decoding a Dataframe using ``'records'`` formatted JSON.+        Note that index labels are not preserved with this encoding.++        >>> df.to_json(orient='records')+        '[{"col 1":"a","col 2":"b"},{"col 1":"c","col 2":"d"}]'++        Encoding/decoding a Dataframe using ``'index'`` formatted JSON:++        >>> df.to_json(orient='index')+        '{"row 1":{"col 1":"a","col 2":"b"},"row 2":{"col 1":"c","col 2":"d"}}'++        Encoding/decoding a Dataframe using ``'columns'`` formatted JSON:++        >>> df.to_json(orient='columns')+        '{"col 1":{"row 1":"a","row 2":"c"},"col 2":{"row 1":"b","row 2":"d"}}'++        Encoding/decoding a Dataframe using ``'values'`` formatted JSON:++        >>> df.to_json(orient='values')+        '[["a","b"],["c","d"]]'++        Encoding with Table Schema++        >>> df.to_json(orient='table')+        '{"schema": {"fields": [{"name": "index", "type": "string"},+                                {"name": "col 1", "type": "string"},+                                {"name": "col 2", "type": "string"}],+                     "primaryKey": "index",+                     "pandas_version": "0.20.0"},+          "data": [{"index": "row 1", "col 1": "a", "col 2": "b"},+                   {"index": "row 2", "col 1": "c", "col 2": "d"}]}'+        """++        from pandas.io import json+        if date_format is None and orient == 'table':+            date_format = 'iso'+        elif date_format is None:+            date_format = 'epoch'+        return json.to_json(path_or_buf=path_or_buf, obj=self, orient=orient,+                            date_format=date_format,+                            double_precision=double_precision,+                            force_ascii=force_ascii, date_unit=date_unit,+                            default_handler=default_handler,+                            lines=lines, compression=compression,+                            index=index)++    def to_hdf(self, path_or_buf, key, **kwargs):+        """+        Write the contained data to an HDF5 file using HDFStore.++        Hierarchical Data Format (HDF) is self-describing, allowing an+        application to interpret the structure and contents of a file with+        no outside information. One HDF file can hold a mix of related objects+        which can be accessed as a group or as individual objects.++        In order to add another DataFrame or Series to an existing HDF file+        please use append mode and a different a key.++        For more information see the :ref:`user guide <io.hdf5>`.++        Parameters+        ----------+        path_or_buf : str or pandas.HDFStore+            File path or HDFStore object.+        key : str+            Identifier for the group in the store.+        mode : {'a', 'w', 'r+'}, default 'a'+            Mode to open file:++            - 'w': write, a new file is created (an existing file with+              the same name would be deleted).+            - 'a': append, an existing file is opened for reading and+              writing, and if the file does not exist it is created.+            - 'r+': similar to 'a', but the file must already exist.+        format : {'fixed', 'table'}, default 'fixed'+            Possible values:++            - 'fixed': Fixed format. Fast writing/reading. Not-appendable,+              nor searchable.+            - 'table': Table format. Write as a PyTables Table structure+              which may perform worse but allow more flexible operations+              like searching / selecting subsets of the data.+        append : bool, default False+            For Table formats, append the input data to the existing.+        data_columns :  list of columns or True, optional+            List of columns to create as indexed data columns for on-disk+            queries, or True to use all columns. By default only the axes+            of the object are indexed. See :ref:`io.hdf5-query-data-columns`.+            Applicable only to format='table'.+        complevel : {0-9}, optional+            Specifies a compression level for data.+            A value of 0 disables compression.+        complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'+            Specifies the compression library to be used.+            As of v0.20.2 these additional compressors for Blosc are supported+            (default if no compressor specified: 'blosc:blosclz'):+            {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',+            'blosc:zlib', 'blosc:zstd'}.+            Specifying a compression library which is not available issues+            a ValueError.+        fletcher32 : bool, default False+            If applying compression use the fletcher32 checksum.+        dropna : bool, default False+            If true, ALL nan rows will not be written to store.+        errors : str, default 'strict'+            Specifies how encoding and decoding errors are to be handled.+            See the errors argument for :func:`open` for a full list+            of options.++        See Also+        --------+        DataFrame.read_hdf : Read from HDF file.+        DataFrame.to_parquet : Write a DataFrame to the binary parquet format.+        DataFrame.to_sql : Write to a sql table.+        DataFrame.to_feather : Write out feather-format for DataFrames.+        DataFrame.to_csv : Write out to a csv file.++        Examples+        --------+        >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]},+        ...                   index=['a', 'b', 'c'])+        >>> df.to_hdf('data.h5', key='df', mode='w')++        We can add another object to the same file:++        >>> s = pd.Series([1, 2, 3, 4])+        >>> s.to_hdf('data.h5', key='s')++        Reading from HDF file:++        >>> pd.read_hdf('data.h5', 'df')+        A  B+        a  1  4+        b  2  5+        c  3  6+        >>> pd.read_hdf('data.h5', 's')+        0    1+        1    2+        2    3+        3    4+        dtype: int64++        Deleting file with data:++        >>> import os+        >>> os.remove('data.h5')++        """+        from pandas.io import pytables+        return pytables.to_hdf(path_or_buf, key, self, **kwargs)++    def to_msgpack(self, path_or_buf=None, encoding='utf-8', **kwargs):+        """+        msgpack (serialize) object to input file path++        THIS IS AN EXPERIMENTAL LIBRARY and the storage format+        may not be stable until a future release.++        Parameters+        ----------+        path : string File path, buffer-like, or None+            if None, return generated string+        append : boolean whether to append to an existing msgpack+            (default is False)+        compress : type of compressor (zlib or blosc), default to None (no+            compression)+        """++        from pandas.io import packers+        return packers.to_msgpack(path_or_buf, self, encoding=encoding,+                                  **kwargs)++    def to_sql(self, name, con, schema=None, if_exists='fail', index=True,+               index_label=None, chunksize=None, dtype=None):+        """+        Write records stored in a DataFrame to a SQL database.++        Databases supported by SQLAlchemy [1]_ are supported. Tables can be+        newly created, appended to, or overwritten.++        Parameters+        ----------+        name : string+            Name of SQL table.+        con : sqlalchemy.engine.Engine or sqlite3.Connection+            Using SQLAlchemy makes it possible to use any DB supported by that+            library. Legacy support is provided for sqlite3.Connection objects.+        schema : string, optional+            Specify the schema (if database flavor supports this). If None, use+            default schema.+        if_exists : {'fail', 'replace', 'append'}, default 'fail'+            How to behave if the table already exists.++            * fail: Raise a ValueError.+            * replace: Drop the table before inserting new values.+            * append: Insert new values to the existing table.++        index : boolean, default True+            Write DataFrame index as a column. Uses `index_label` as the column+            name in the table.+        index_label : string or sequence, default None+            Column label for index column(s). If None is given (default) and+            `index` is True, then the index names are used.+            A sequence should be given if the DataFrame uses MultiIndex.+        chunksize : int, optional+            Rows will be written in batches of this size at a time. By default,+            all rows will be written at once.+        dtype : dict, optional+            Specifying the datatype for columns. The keys should be the column+            names and the values should be the SQLAlchemy types or strings for+            the sqlite3 legacy mode.++        Raises+        ------+        ValueError+            When the table already exists and `if_exists` is 'fail' (the+            default).++        See Also+        --------+        pandas.read_sql : read a DataFrame from a table++        References+        ----------+        .. [1] http://docs.sqlalchemy.org+        .. [2] https://www.python.org/dev/peps/pep-0249/++        Examples+        --------++        Create an in-memory SQLite database.++        >>> from sqlalchemy import create_engine+        >>> engine = create_engine('sqlite://', echo=False)++        Create a table from scratch with 3 rows.++        >>> df = pd.DataFrame({'name' : ['User 1', 'User 2', 'User 3']})+        >>> df+             name+        0  User 1+        1  User 2+        2  User 3++        >>> df.to_sql('users', con=engine)+        >>> engine.execute("SELECT * FROM users").fetchall()+        [(0, 'User 1'), (1, 'User 2'), (2, 'User 3')]++        >>> df1 = pd.DataFrame({'name' : ['User 4', 'User 5']})+        >>> df1.to_sql('users', con=engine, if_exists='append')+        >>> engine.execute("SELECT * FROM users").fetchall()+        [(0, 'User 1'), (1, 'User 2'), (2, 'User 3'),+         (0, 'User 4'), (1, 'User 5')]++        Overwrite the table with just ``df1``.++        >>> df1.to_sql('users', con=engine, if_exists='replace',+        ...            index_label='id')+        >>> engine.execute("SELECT * FROM users").fetchall()+        [(0, 'User 4'), (1, 'User 5')]++        Specify the dtype (especially useful for integers with missing values).+        Notice that while pandas is forced to store the data as floating point,+        the database supports nullable integers. When fetching the data with+        Python, we get back integer scalars.++        >>> df = pd.DataFrame({"A": [1, None, 2]})+        >>> df+             A+        0  1.0+        1  NaN+        2  2.0++        >>> from sqlalchemy.types import Integer+        >>> df.to_sql('integers', con=engine, index=False,+        ...           dtype={"A": Integer()})++        >>> engine.execute("SELECT * FROM integers").fetchall()+        [(1,), (None,), (2,)]+        """+        from pandas.io import sql+        sql.to_sql(self, name, con, schema=schema, if_exists=if_exists,+                   index=index, index_label=index_label, chunksize=chunksize,+                   dtype=dtype)++    def to_pickle(self, path, compression='infer',+                  protocol=pkl.HIGHEST_PROTOCOL):+        """+        Pickle (serialize) object to file.++        Parameters+        ----------+        path : str+            File path where the pickled object will be stored.+        compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None}, \+        default 'infer'+            A string representing the compression to use in the output file. By+            default, infers from the file extension in specified path.++            .. versionadded:: 0.20.0+        protocol : int+            Int which indicates which protocol should be used by the pickler,+            default HIGHEST_PROTOCOL (see [1]_ paragraph 12.1.2). The possible+            values for this parameter depend on the version of Python. For+            Python 2.x, possible values are 0, 1, 2. For Python>=3.0, 3 is a+            valid value. For Python >= 3.4, 4 is a valid value. A negative+            value for the protocol parameter is equivalent to setting its value+            to HIGHEST_PROTOCOL.++            .. [1] https://docs.python.org/3/library/pickle.html+            .. versionadded:: 0.21.0++        See Also+        --------+        read_pickle : Load pickled pandas object (or any object) from file.+        DataFrame.to_hdf : Write DataFrame to an HDF5 file.+        DataFrame.to_sql : Write DataFrame to a SQL database.+        DataFrame.to_parquet : Write a DataFrame to the binary parquet format.++        Examples+        --------+        >>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)})+        >>> original_df+           foo  bar+        0    0    5+        1    1    6+        2    2    7+        3    3    8+        4    4    9+        >>> original_df.to_pickle("./dummy.pkl")++        >>> unpickled_df = pd.read_pickle("./dummy.pkl")+        >>> unpickled_df+           foo  bar+        0    0    5+        1    1    6+        2    2    7+        3    3    8+        4    4    9++        >>> import os+        >>> os.remove("./dummy.pkl")+        """+        from pandas.io.pickle import to_pickle+        return to_pickle(self, path, compression=compression,+                         protocol=protocol)++    def to_clipboard(self, excel=True, sep=None, **kwargs):+        r"""+        Copy object to the system clipboard.++        Write a text representation of object to the system clipboard.+        This can be pasted into Excel, for example.++        Parameters+        ----------+        excel : bool, default True+            - True, use the provided separator, writing in a csv format for+              allowing easy pasting into excel.+            - False, write a string representation of the object to the+              clipboard.++        sep : str, default ``'\t'``+            Field delimiter.+        **kwargs+            These parameters will be passed to DataFrame.to_csv.++        See Also+        --------+        DataFrame.to_csv : Write a DataFrame to a comma-separated values+            (csv) file.+        read_clipboard : Read text from clipboard and pass to read_table.++        Notes+        -----+        Requirements for your platform.++          - Linux : `xclip`, or `xsel` (with `gtk` or `PyQt4` modules)+          - Windows : none+          - OS X : none++        Examples+        --------+        Copy the contents of a DataFrame to the clipboard.++        >>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=['A', 'B', 'C'])+        >>> df.to_clipboard(sep=',')+        ... # Wrote the following to the system clipboard:+        ... # ,A,B,C+        ... # 0,1,2,3+        ... # 1,4,5,6++        We can omit the the index by passing the keyword `index` and setting+        it to false.++        >>> df.to_clipboard(sep=',', index=False)+        ... # Wrote the following to the system clipboard:+        ... # A,B,C+        ... # 1,2,3+        ... # 4,5,6+        """+        from pandas.io import clipboards+        clipboards.to_clipboard(self, excel=excel, sep=sep, **kwargs)++    def to_xarray(self):+        """+        Return an xarray object from the pandas object.++        Returns+        -------+        a DataArray for a Series+        a Dataset for a DataFrame+        a DataArray for higher dims++        Examples+        --------+        >>> df = pd.DataFrame({'A' : [1, 1, 2],+                               'B' : ['foo', 'bar', 'foo'],+                               'C' : np.arange(4.,7)})+        >>> df+           A    B    C+        0  1  foo  4.0+        1  1  bar  5.0+        2  2  foo  6.0++        >>> df.to_xarray()+        <xarray.Dataset>+        Dimensions:  (index: 3)+        Coordinates:+          * index    (index) int64 0 1 2+        Data variables:+            A        (index) int64 1 1 2+            B        (index) object 'foo' 'bar' 'foo'+            C        (index) float64 4.0 5.0 6.0++        >>> df = pd.DataFrame({'A' : [1, 1, 2],+                               'B' : ['foo', 'bar', 'foo'],+                               'C' : np.arange(4.,7)}+                             ).set_index(['B','A'])+        >>> df+                 C+        B   A+        foo 1  4.0+        bar 1  5.0+        foo 2  6.0++        >>> df.to_xarray()+        <xarray.Dataset>+        Dimensions:  (A: 2, B: 2)+        Coordinates:+          * B        (B) object 'bar' 'foo'+          * A        (A) int64 1 2+        Data variables:+            C        (B, A) float64 5.0 nan 4.0 6.0++        >>> p = pd.Panel(np.arange(24).reshape(4,3,2),+                         items=list('ABCD'),+                         major_axis=pd.date_range('20130101', periods=3),+                         minor_axis=['first', 'second'])+        >>> p+        <class 'pandas.core.panel.Panel'>+        Dimensions: 4 (items) x 3 (major_axis) x 2 (minor_axis)+        Items axis: A to D+        Major_axis axis: 2013-01-01 00:00:00 to 2013-01-03 00:00:00+        Minor_axis axis: first to second++        >>> p.to_xarray()+        <xarray.DataArray (items: 4, major_axis: 3, minor_axis: 2)>+        array([[[ 0,  1],+                [ 2,  3],+                [ 4,  5]],+               [[ 6,  7],+                [ 8,  9],+                [10, 11]],+               [[12, 13],+                [14, 15],+                [16, 17]],+               [[18, 19],+                [20, 21],+                [22, 23]]])+        Coordinates:+          * items       (items) object 'A' 'B' 'C' 'D'+          * major_axis  (major_axis) datetime64[ns] 2013-01-01 2013-01-02 2013-01-03  # noqa+          * minor_axis  (minor_axis) object 'first' 'second'++        Notes+        -----+        See the `xarray docs <http://xarray.pydata.org/en/stable/>`__+        """++        try:+            import xarray+        except ImportError:+            # Give a nice error message+            raise ImportError("the xarray library is not installed\n"+                              "you can install via conda\n"+                              "conda install xarray\n"+                              "or via pip\n"+                              "pip install xarray\n")++        if self.ndim == 1:+            return xarray.DataArray.from_series(self)+        elif self.ndim == 2:+            return xarray.Dataset.from_dataframe(self)++        # > 2 dims+        coords = [(a, self._get_axis(a)) for a in self._AXIS_ORDERS]+        return xarray.DataArray(self,+                                coords=coords,+                                )++    _shared_docs['to_latex'] = r"""+        Render an object to a tabular environment table. You can splice+        this into a LaTeX document. Requires \\usepackage{booktabs}.++        .. versionchanged:: 0.20.2+           Added to Series++        `to_latex`-specific options:++        bold_rows : boolean, default False+            Make the row labels bold in the output+        column_format : str, default None+            The columns format as specified in `LaTeX table format+            <https://en.wikibooks.org/wiki/LaTeX/Tables>`__ e.g 'rcl' for 3+            columns+        longtable : boolean, default will be read from the pandas config module+            Default: False.+            Use a longtable environment instead of tabular. Requires adding+            a \\usepackage{longtable} to your LaTeX preamble.+        escape : boolean, default will be read from the pandas config module+            Default: True.+            When set to False prevents from escaping latex special+            characters in column names.+        encoding : str, default None+            A string representing the encoding to use in the output file,+            defaults to 'ascii' on Python 2 and 'utf-8' on Python 3.+        decimal : string, default '.'+            Character recognized as decimal separator, e.g. ',' in Europe.++            .. versionadded:: 0.18.0++        multicolumn : boolean, default True+            Use \multicolumn to enhance MultiIndex columns.+            The default will be read from the config module.++            .. versionadded:: 0.20.0++        multicolumn_format : str, default 'l'+            The alignment for multicolumns, similar to `column_format`+            The default will be read from the config module.++            .. versionadded:: 0.20.0++        multirow : boolean, default False+            Use \multirow to enhance MultiIndex rows.+            Requires adding a \\usepackage{multirow} to your LaTeX preamble.+            Will print centered labels (instead of top-aligned)+            across the contained rows, separating groups via clines.+            The default will be read from the pandas config module.++            .. versionadded:: 0.20.0+            """++    @Substitution(header='Write out the column names. If a list of strings '+                         'is given, it is assumed to be aliases for the '+                         'column names.')+    @Appender(_shared_docs['to_latex'] % _shared_doc_kwargs)+    def to_latex(self, buf=None, columns=None, col_space=None, header=True,+                 index=True, na_rep='NaN', formatters=None, float_format=None,+                 sparsify=None, index_names=True, bold_rows=False,+                 column_format=None, longtable=None, escape=None,+                 encoding=None, decimal='.', multicolumn=None,+                 multicolumn_format=None, multirow=None):+        # Get defaults from the pandas config+        if self.ndim == 1:+            self = self.to_frame()+        if longtable is None:+            longtable = config.get_option("display.latex.longtable")+        if escape is None:+            escape = config.get_option("display.latex.escape")+        if multicolumn is None:+            multicolumn = config.get_option("display.latex.multicolumn")+        if multicolumn_format is None:+            multicolumn_format = config.get_option(+                "display.latex.multicolumn_format")+        if multirow is None:+            multirow = config.get_option("display.latex.multirow")++        formatter = DataFrameFormatter(self, buf=buf, columns=columns,+                                       col_space=col_space, na_rep=na_rep,+                                       header=header, index=index,+                                       formatters=formatters,+                                       float_format=float_format,+                                       bold_rows=bold_rows,+                                       sparsify=sparsify,+                                       index_names=index_names,+                                       escape=escape, decimal=decimal)+        formatter.to_latex(column_format=column_format, longtable=longtable,+                           encoding=encoding, multicolumn=multicolumn,+                           multicolumn_format=multicolumn_format,+                           multirow=multirow)++        if buf is None:+            return formatter.buf.getvalue()++    # ----------------------------------------------------------------------+    # Fancy Indexing++    @classmethod+    def _create_indexer(cls, name, indexer):+        """Create an indexer like _name in the class."""+        if getattr(cls, name, None) is None:+            _indexer = functools.partial(indexer, name)+            setattr(cls, name, property(_indexer, doc=indexer.__doc__))++    def get(self, key, default=None):+        """+        Get item from object for given key (DataFrame column, Panel slice,+        etc.). Returns default value if not found.++        Parameters+        ----------+        key : object++        Returns+        -------+        value : same type as items contained in object+        """+        try:+            return self[key]+        except (KeyError, ValueError, IndexError):+            return default++    def __getitem__(self, item):+        return self._get_item_cache(item)++    def _get_item_cache(self, item):+        """Return the cached item, item represents a label indexer."""+        cache = self._item_cache+        res = cache.get(item)+        if res is None:+            values = self._data.get(item)+            res = self._box_item_values(item, values)+            cache[item] = res+            res._set_as_cached(item, self)++            # for a chain+            res._is_copy = self._is_copy+        return res++    def _set_as_cached(self, item, cacher):+        """Set the _cacher attribute on the calling object with a weakref to+        cacher.+        """+        self._cacher = (item, weakref.ref(cacher))++    def _reset_cacher(self):+        """Reset the cacher."""+        if hasattr(self, '_cacher'):+            del self._cacher++    def _iget_item_cache(self, item):+        """Return the cached item, item represents a positional indexer."""+        ax = self._info_axis+        if ax.is_unique:+            lower = self._get_item_cache(ax[item])+        else:+            lower = self._take(item, axis=self._info_axis_number)+        return lower++    def _box_item_values(self, key, values):+        raise com.AbstractMethodError(self)++    def _maybe_cache_changed(self, item, value):+        """The object has called back to us saying maybe it has changed.+        """+        self._data.set(item, value, check=False)++    @property+    def _is_cached(self):+        """Return boolean indicating if self is cached or not."""+        return getattr(self, '_cacher', None) is not None++    def _get_cacher(self):+        """return my cacher or None"""+        cacher = getattr(self, '_cacher', None)+        if cacher is not None:+            cacher = cacher[1]()+        return cacher++    @property+    def _is_view(self):+        """Return boolean indicating if self is view of another array """+        return self._data.is_view++    def _maybe_update_cacher(self, clear=False, verify_is_copy=True):+        """+        See if we need to update our parent cacher if clear, then clear our+        cache.++        Parameters+        ----------+        clear : boolean, default False+            clear the item cache+        verify_is_copy : boolean, default True+            provide is_copy checks++        """++        cacher = getattr(self, '_cacher', None)+        if cacher is not None:+            ref = cacher[1]()++            # we are trying to reference a dead referant, hence+            # a copy+            if ref is None:+                del self._cacher+            else:+                try:+                    ref._maybe_cache_changed(cacher[0], self)+                except Exception:+                    pass++        if verify_is_copy:+            self._check_setitem_copy(stacklevel=5, t='referant')++        if clear:+            self._clear_item_cache()++    def _clear_item_cache(self, i=None):+        if i is not None:+            self._item_cache.pop(i, None)+        else:+            self._item_cache.clear()++    def _slice(self, slobj, axis=0, kind=None):+        """+        Construct a slice of this container.++        kind parameter is maintained for compatibility with Series slicing.+        """+        axis = self._get_block_manager_axis(axis)+        result = self._constructor(self._data.get_slice(slobj, axis=axis))+        result = result.__finalize__(self)++        # this could be a view+        # but only in a single-dtyped view slicable case+        is_copy = axis != 0 or result._is_view+        result._set_is_copy(self, copy=is_copy)+        return result++    def _set_item(self, key, value):+        self._data.set(key, value)+        self._clear_item_cache()++    def _set_is_copy(self, ref=None, copy=True):+        if not copy:+            self._is_copy = None+        else:+            if ref is not None:+                self._is_copy = weakref.ref(ref)+            else:+                self._is_copy = None++    def _check_is_chained_assignment_possible(self):+        """+        Check if we are a view, have a cacher, and are of mixed type.+        If so, then force a setitem_copy check.++        Should be called just near setting a value++        Will return a boolean if it we are a view and are cached, but a+        single-dtype meaning that the cacher should be updated following+        setting.+        """+        if self._is_view and self._is_cached:+            ref = self._get_cacher()+            if ref is not None and ref._is_mixed_type:+                self._check_setitem_copy(stacklevel=4, t='referant',+                                         force=True)+            return True+        elif self._is_copy:+            self._check_setitem_copy(stacklevel=4, t='referant')+        return False++    def _check_setitem_copy(self, stacklevel=4, t='setting', force=False):+        """++        Parameters+        ----------+        stacklevel : integer, default 4+           the level to show of the stack when the error is output+        t : string, the type of setting error+        force : boolean, default False+           if True, then force showing an error++        validate if we are doing a settitem on a chained copy.++        If you call this function, be sure to set the stacklevel such that the+        user will see the error *at the level of setting*++        It is technically possible to figure out that we are setting on+        a copy even WITH a multi-dtyped pandas object. In other words, some+        blocks may be views while other are not. Currently _is_view will ALWAYS+        return False for multi-blocks to avoid having to handle this case.++        df = DataFrame(np.arange(0,9), columns=['count'])+        df['group'] = 'b'++        # This technically need not raise SettingWithCopy if both are view+        # (which is not # generally guaranteed but is usually True.  However,+        # this is in general not a good practice and we recommend using .loc.+        df.iloc[0:5]['group'] = 'a'++        """++        if force or self._is_copy:++            value = config.get_option('mode.chained_assignment')+            if value is None:+                return++            # see if the copy is not actually referred; if so, then dissolve+            # the copy weakref+            try:+                gc.collect(2)+                if not gc.get_referents(self._is_copy()):+                    self._is_copy = None+                    return+            except Exception:+                pass++            # we might be a false positive+            try:+                if self._is_copy().shape == self.shape:+                    self._is_copy = None+                    return+            except Exception:+                pass++            # a custom message+            if isinstance(self._is_copy, string_types):+                t = self._is_copy++            elif t == 'referant':+                t = ("\n"+                     "A value is trying to be set on a copy of a slice from a "+                     "DataFrame\n\n"+                     "See the caveats in the documentation: "+                     "http://pandas.pydata.org/pandas-docs/stable/"+                     "indexing.html#indexing-view-versus-copy"+                     )++            else:+                t = ("\n"+                     "A value is trying to be set on a copy of a slice from a "+                     "DataFrame.\n"+                     "Try using .loc[row_indexer,col_indexer] = value "+                     "instead\n\nSee the caveats in the documentation: "+                     "http://pandas.pydata.org/pandas-docs/stable/"+                     "indexing.html#indexing-view-versus-copy"+                     )++            if value == 'raise':+                raise com.SettingWithCopyError(t)+            elif value == 'warn':+                warnings.warn(t, com.SettingWithCopyWarning,+                              stacklevel=stacklevel)++    def __delitem__(self, key):+        """+        Delete item+        """+        deleted = False++        maybe_shortcut = False+        if hasattr(self, 'columns') and isinstance(self.columns, MultiIndex):+            try:+                maybe_shortcut = key not in self.columns._engine+            except TypeError:+                pass++        if maybe_shortcut:+            # Allow shorthand to delete all columns whose first len(key)+            # elements match key:+            if not isinstance(key, tuple):+                key = (key, )+            for col in self.columns:+                if isinstance(col, tuple) and col[:len(key)] == key:+                    del self[col]+                    deleted = True+        if not deleted:+            # If the above loop ran and didn't delete anything because+            # there was no match, this call should raise the appropriate+            # exception:+            self._data.delete(key)++        # delete from the caches+        try:+            del self._item_cache[key]+        except KeyError:+            pass++    _shared_docs['_take'] = """+        Return the elements in the given *positional* indices along an axis.++        This means that we are not indexing according to actual values in+        the index attribute of the object. We are indexing according to the+        actual position of the element in the object.++        This is the internal version of ``.take()`` and will contain a wider+        selection of parameters useful for internal use but not as suitable+        for public usage.++        Parameters+        ----------+        indices : array-like+            An array of ints indicating which positions to take.+        axis : int, default 0+            The axis on which to select elements. "0" means that we are+            selecting rows, "1" means that we are selecting columns, etc.+        is_copy : bool, default True+            Whether to return a copy of the original object or not.++        Returns+        -------+        taken : same type as caller+            An array-like containing the elements taken from the object.++        See Also+        --------+        numpy.ndarray.take+        numpy.take+        """++    @Appender(_shared_docs['_take'])+    def _take(self, indices, axis=0, is_copy=True):+        self._consolidate_inplace()++        new_data = self._data.take(indices,+                                   axis=self._get_block_manager_axis(axis),+                                   verify=True)+        result = self._constructor(new_data).__finalize__(self)++        # Maybe set copy if we didn't actually change the index.+        if is_copy:+            if not result._get_axis(axis).equals(self._get_axis(axis)):+                result._set_is_copy(self)++        return result++    _shared_docs['take'] = """+        Return the elements in the given *positional* indices along an axis.++        This means that we are not indexing according to actual values in+        the index attribute of the object. We are indexing according to the+        actual position of the element in the object.++        Parameters+        ----------+        indices : array-like+            An array of ints indicating which positions to take.+        axis : {0 or 'index', 1 or 'columns', None}, default 0+            The axis on which to select elements. ``0`` means that we are+            selecting rows, ``1`` means that we are selecting columns.+        convert : bool, default True+            Whether to convert negative indices into positive ones.+            For example, ``-1`` would map to the ``len(axis) - 1``.+            The conversions are similar to the behavior of indexing a+            regular Python list.++            .. deprecated:: 0.21.0+               In the future, negative indices will always be converted.++        is_copy : bool, default True+            Whether to return a copy of the original object or not.+        **kwargs+            For compatibility with :meth:`numpy.take`. Has no effect on the+            output.++        Returns+        -------+        taken : same type as caller+            An array-like containing the elements taken from the object.++        See Also+        --------+        DataFrame.loc : Select a subset of a DataFrame by labels.+        DataFrame.iloc : Select a subset of a DataFrame by positions.+        numpy.take : Take elements from an array along an axis.++        Examples+        --------+        >>> df = pd.DataFrame([('falcon', 'bird',    389.0),+        ...                    ('parrot', 'bird',     24.0),+        ...                    ('lion',   'mammal',   80.5),+        ...                    ('monkey', 'mammal', np.nan)],+        ...                    columns=['name', 'class', 'max_speed'],+        ...                    index=[0, 2, 3, 1])+        >>> df+             name   class  max_speed+        0  falcon    bird      389.0+        2  parrot    bird       24.0+        3    lion  mammal       80.5+        1  monkey  mammal        NaN++        Take elements at positions 0 and 3 along the axis 0 (default).++        Note how the actual indices selected (0 and 1) do not correspond to+        our selected indices 0 and 3. That's because we are selecting the 0th+        and 3rd rows, not rows whose indices equal 0 and 3.++        >>> df.take([0, 3])+             name   class  max_speed+        0  falcon    bird      389.0+        1  monkey  mammal        NaN++        Take elements at indices 1 and 2 along the axis 1 (column selection).++        >>> df.take([1, 2], axis=1)+            class  max_speed+        0    bird      389.0+        2    bird       24.0+        3  mammal       80.5+        1  mammal        NaN++        We may take elements using negative integers for positive indices,+        starting from the end of the object, just like with Python lists.++        >>> df.take([-1, -2])+             name   class  max_speed+        1  monkey  mammal        NaN+        3    lion  mammal       80.5+        """++    @Appender(_shared_docs['take'])+    def take(self, indices, axis=0, convert=None, is_copy=True, **kwargs):+        if convert is not None:+            msg = ("The 'convert' parameter is deprecated "+                   "and will be removed in a future version.")+            warnings.warn(msg, FutureWarning, stacklevel=2)++        nv.validate_take(tuple(), kwargs)+        return self._take(indices, axis=axis, is_copy=is_copy)++    def xs(self, key, axis=0, level=None, drop_level=True):+        """+        Returns a cross-section (row(s) or column(s)) from the+        Series/DataFrame. Defaults to cross-section on the rows (axis=0).++        Parameters+        ----------+        key : object+            Some label contained in the index, or partially in a MultiIndex+        axis : int, default 0+            Axis to retrieve cross-section on+        level : object, defaults to first n levels (n=1 or len(key))+            In case of a key partially contained in a MultiIndex, indicate+            which levels are used. Levels can be referred by label or position.+        drop_level : boolean, default True+            If False, returns object with same levels as self.++        Examples+        --------+        >>> df+           A  B  C+        a  4  5  2+        b  4  0  9+        c  9  7  3+        >>> df.xs('a')+        A    4+        B    5+        C    2+        Name: a+        >>> df.xs('C', axis=1)+        a    2+        b    9+        c    3+        Name: C++        >>> df+                            A  B  C  D+        first second third+        bar   one    1      4  1  8  9+              two    1      7  5  5  0+        baz   one    1      6  6  8  0+              three  2      5  3  5  3+        >>> df.xs(('baz', 'three'))+               A  B  C  D+        third+        2      5  3  5  3+        >>> df.xs('one', level=1)+                     A  B  C  D+        first third+        bar   1      4  1  8  9+        baz   1      6  6  8  0+        >>> df.xs(('baz', 2), level=[0, 'third'])+                A  B  C  D+        second+        three   5  3  5  3++        Returns+        -------+        xs : Series or DataFrame++        Notes+        -----+        xs is only for getting, not setting values.++        MultiIndex Slicers is a generic way to get/set values on any level or+        levels.  It is a superset of xs functionality, see+        :ref:`MultiIndex Slicers <advanced.mi_slicers>`++        """+        axis = self._get_axis_number(axis)+        labels = self._get_axis(axis)+        if level is not None:+            loc, new_ax = labels.get_loc_level(key, level=level,+                                               drop_level=drop_level)++            # create the tuple of the indexer+            indexer = [slice(None)] * self.ndim+            indexer[axis] = loc+            indexer = tuple(indexer)++            result = self.iloc[indexer]+            setattr(result, result._get_axis_name(axis), new_ax)+            return result++        if axis == 1:+            return self[key]++        self._consolidate_inplace()++        index = self.index+        if isinstance(index, MultiIndex):+            loc, new_index = self.index.get_loc_level(key,+                                                      drop_level=drop_level)+        else:+            loc = self.index.get_loc(key)++            if isinstance(loc, np.ndarray):+                if loc.dtype == np.bool_:+                    inds, = loc.nonzero()+                    return self._take(inds, axis=axis)+                else:+                    return self._take(loc, axis=axis)++            if not is_scalar(loc):+                new_index = self.index[loc]++        if is_scalar(loc):+            new_values = self._data.fast_xs(loc)++            # may need to box a datelike-scalar+            #+            # if we encounter an array-like and we only have 1 dim+            # that means that their are list/ndarrays inside the Series!+            # so just return them (GH 6394)+            if not is_list_like(new_values) or self.ndim == 1:+                return com.maybe_box_datetimelike(new_values)++            result = self._constructor_sliced(+                new_values, index=self.columns,+                name=self.index[loc], dtype=new_values.dtype)++        else:+            result = self.iloc[loc]+            result.index = new_index++        # this could be a view+        # but only in a single-dtyped view slicable case+        result._set_is_copy(self, copy=not result._is_view)+        return result++    _xs = xs++    def select(self, crit, axis=0):+        """Return data corresponding to axis labels matching criteria++        .. deprecated:: 0.21.0+            Use df.loc[df.index.map(crit)] to select via labels++        Parameters+        ----------+        crit : function+            To be called on each index (label). Should return True or False+        axis : int++        Returns+        -------+        selection : same type as caller+        """+        warnings.warn("'select' is deprecated and will be removed in a "+                      "future release. You can use "+                      ".loc[labels.map(crit)] as a replacement",+                      FutureWarning, stacklevel=2)++        axis = self._get_axis_number(axis)+        axis_name = self._get_axis_name(axis)+        axis_values = self._get_axis(axis)++        if len(axis_values) > 0:+            new_axis = axis_values[+                np.asarray([bool(crit(label)) for label in axis_values])]+        else:+            new_axis = axis_values++        return self.reindex(**{axis_name: new_axis})++    def reindex_like(self, other, method=None, copy=True, limit=None,+                     tolerance=None):+        """Return an object with matching indices to myself.++        Parameters+        ----------+        other : Object+        method : string or None+        copy : boolean, default True+        limit : int, default None+            Maximum number of consecutive labels to fill for inexact matches.+        tolerance : optional+            Maximum distance between labels of the other object and this+            object for inexact matches. Can be list-like.++            .. versionadded:: 0.21.0 (list-like tolerance)++        Notes+        -----+        Like calling s.reindex(index=other.index, columns=other.columns,+                               method=...)++        Returns+        -------+        reindexed : same as input+        """+        d = other._construct_axes_dict(axes=self._AXIS_ORDERS, method=method,+                                       copy=copy, limit=limit,+                                       tolerance=tolerance)++        return self.reindex(**d)++    def drop(self, labels=None, axis=0, index=None, columns=None, level=None,+             inplace=False, errors='raise'):++        inplace = validate_bool_kwarg(inplace, 'inplace')++        if labels is not None:+            if index is not None or columns is not None:+                raise ValueError("Cannot specify both 'labels' and "+                                 "'index'/'columns'")+            axis_name = self._get_axis_name(axis)+            axes = {axis_name: labels}+        elif index is not None or columns is not None:+            axes, _ = self._construct_axes_from_arguments((index, columns), {})+        else:+            raise ValueError("Need to specify at least one of 'labels', "+                             "'index' or 'columns'")++        obj = self++        for axis, labels in axes.items():+            if labels is not None:+                obj = obj._drop_axis(labels, axis, level=level, errors=errors)++        if inplace:+            self._update_inplace(obj)+        else:+            return obj++    def _drop_axis(self, labels, axis, level=None, errors='raise'):+        """+        Drop labels from specified axis. Used in the ``drop`` method+        internally.++        Parameters+        ----------+        labels : single label or list-like+        axis : int or axis name+        level : int or level name, default None+            For MultiIndex+        errors : {'ignore', 'raise'}, default 'raise'+            If 'ignore', suppress error and existing labels are dropped.++        """+        axis = self._get_axis_number(axis)+        axis_name = self._get_axis_name(axis)+        axis = self._get_axis(axis)++        if axis.is_unique:+            if level is not None:+                if not isinstance(axis, MultiIndex):+                    raise AssertionError('axis must be a MultiIndex')+                new_axis = axis.drop(labels, level=level, errors=errors)+            else:+                new_axis = axis.drop(labels, errors=errors)+            result = self.reindex(**{axis_name: new_axis})++        # Case for non-unique axis+        else:+            labels = ensure_object(com.index_labels_to_array(labels))+            if level is not None:+                if not isinstance(axis, MultiIndex):+                    raise AssertionError('axis must be a MultiIndex')+                indexer = ~axis.get_level_values(level).isin(labels)++                # GH 18561 MultiIndex.drop should raise if label is absent+                if errors == 'raise' and indexer.all():+                    raise KeyError('{} not found in axis'.format(labels))+            else:+                indexer = ~axis.isin(labels)+                # Check if label doesn't exist along axis+                labels_missing = (axis.get_indexer_for(labels) == -1).any()+                if errors == 'raise' and labels_missing:+                    raise KeyError('{} not found in axis'.format(labels))++            slicer = [slice(None)] * self.ndim+            slicer[self._get_axis_number(axis_name)] = indexer++            result = self.loc[tuple(slicer)]++        return result++    def _update_inplace(self, result, verify_is_copy=True):+        """+        Replace self internals with result.++        Parameters+        ----------+        verify_is_copy : boolean, default True+            provide is_copy checks++        """+        # NOTE: This does *not* call __finalize__ and that's an explicit+        # decision that we may revisit in the future.++        self._reset_cache()+        self._clear_item_cache()+        self._data = getattr(result, '_data', result)+        self._maybe_update_cacher(verify_is_copy=verify_is_copy)++    def add_prefix(self, prefix):+        """+        Prefix labels with string `prefix`.++        For Series, the row labels are prefixed.+        For DataFrame, the column labels are prefixed.++        Parameters+        ----------+        prefix : str+            The string to add before each label.++        Returns+        -------+        Series or DataFrame+            New Series or DataFrame with updated labels.++        See Also+        --------+        Series.add_suffix: Suffix row labels with string `suffix`.+        DataFrame.add_suffix: Suffix column labels with string `suffix`.++        Examples+        --------+        >>> s = pd.Series([1, 2, 3, 4])+        >>> s+        0    1+        1    2+        2    3+        3    4+        dtype: int64++        >>> s.add_prefix('item_')+        item_0    1+        item_1    2+        item_2    3+        item_3    4+        dtype: int64++        >>> df = pd.DataFrame({'A': [1, 2, 3, 4],  'B': [3, 4, 5, 6]})+        >>> df+           A  B+        0  1  3+        1  2  4+        2  3  5+        3  4  6++        >>> df.add_prefix('col_')+             col_A  col_B+        0       1       3+        1       2       4+        2       3       5+        3       4       6+        """+        new_data = self._data.add_prefix(prefix)+        return self._constructor(new_data).__finalize__(self)++    def add_suffix(self, suffix):+        """+        Suffix labels with string `suffix`.++        For Series, the row labels are suffixed.+        For DataFrame, the column labels are suffixed.++        Parameters+        ----------+        suffix : str+            The string to add after each label.++        Returns+        -------+        Series or DataFrame+            New Series or DataFrame with updated labels.++        See Also+        --------+        Series.add_prefix: Prefix row labels with string `prefix`.+        DataFrame.add_prefix: Prefix column labels with string `prefix`.++        Examples+        --------+        >>> s = pd.Series([1, 2, 3, 4])+        >>> s+        0    1+        1    2+        2    3+        3    4+        dtype: int64++        >>> s.add_suffix('_item')+        0_item    1+        1_item    2+        2_item    3+        3_item    4+        dtype: int64++        >>> df = pd.DataFrame({'A': [1, 2, 3, 4],  'B': [3, 4, 5, 6]})+        >>> df+           A  B+        0  1  3+        1  2  4+        2  3  5+        3  4  6++        >>> df.add_suffix('_col')+             A_col  B_col+        0       1       3+        1       2       4+        2       3       5+        3       4       6+        """+        new_data = self._data.add_suffix(suffix)+        return self._constructor(new_data).__finalize__(self)++    _shared_docs['sort_values'] = """+        Sort by the values along either axis++        Parameters+        ----------%(optional_by)s+        axis : %(axes_single_arg)s, default 0+             Axis to be sorted+        ascending : bool or list of bool, default True+             Sort ascending vs. descending. Specify list for multiple sort+             orders.  If this is a list of bools, must match the length of+             the by.+        inplace : bool, default False+             if True, perform operation in-place+        kind : {'quicksort', 'mergesort', 'heapsort'}, default 'quicksort'+             Choice of sorting algorithm. See also ndarray.np.sort for more+             information.  `mergesort` is the only stable algorithm. For+             DataFrames, this option is only applied when sorting on a single+             column or label.+        na_position : {'first', 'last'}, default 'last'+             `first` puts NaNs at the beginning, `last` puts NaNs at the end++        Returns+        -------+        sorted_obj : %(klass)s++        Examples+        --------+        >>> df = pd.DataFrame({+        ...     'col1' : ['A', 'A', 'B', np.nan, 'D', 'C'],+        ...     'col2' : [2, 1, 9, 8, 7, 4],+        ...     'col3': [0, 1, 9, 4, 2, 3],+        ... })+        >>> df+            col1 col2 col3+        0   A    2    0+        1   A    1    1+        2   B    9    9+        3   NaN  8    4+        4   D    7    2+        5   C    4    3++        Sort by col1++        >>> df.sort_values(by=['col1'])+            col1 col2 col3+        0   A    2    0+        1   A    1    1+        2   B    9    9+        5   C    4    3+        4   D    7    2+        3   NaN  8    4++        Sort by multiple columns++        >>> df.sort_values(by=['col1', 'col2'])+            col1 col2 col3+        1   A    1    1+        0   A    2    0+        2   B    9    9+        5   C    4    3+        4   D    7    2+        3   NaN  8    4++        Sort Descending++        >>> df.sort_values(by='col1', ascending=False)+            col1 col2 col3+        4   D    7    2+        5   C    4    3+        2   B    9    9+        0   A    2    0+        1   A    1    1+        3   NaN  8    4++        Putting NAs first++        >>> df.sort_values(by='col1', ascending=False, na_position='first')+            col1 col2 col3+        3   NaN  8    4+        4   D    7    2+        5   C    4    3+        2   B    9    9+        0   A    2    0+        1   A    1    1+        """++    def sort_values(self, by=None, axis=0, ascending=True, inplace=False,+                    kind='quicksort', na_position='last'):+        """+        NOT IMPLEMENTED: do not call this method, as sorting values is not+        supported for Panel objects and will raise an error.+        """+        raise NotImplementedError("sort_values has not been implemented "+                                  "on Panel or Panel4D objects.")++    _shared_docs['sort_index'] = """+        Sort object by labels (along an axis)++        Parameters+        ----------+        axis : %(axes)s to direct sorting+        level : int or level name or list of ints or list of level names+            if not None, sort on values in specified index level(s)+        ascending : boolean, default True+            Sort ascending vs. descending+        inplace : bool, default False+            if True, perform operation in-place+        kind : {'quicksort', 'mergesort', 'heapsort'}, default 'quicksort'+             Choice of sorting algorithm. See also ndarray.np.sort for more+             information.  `mergesort` is the only stable algorithm. For+             DataFrames, this option is only applied when sorting on a single+             column or label.+        na_position : {'first', 'last'}, default 'last'+             `first` puts NaNs at the beginning, `last` puts NaNs at the end.+             Not implemented for MultiIndex.+        sort_remaining : bool, default True+            if true and sorting by level and index is multilevel, sort by other+            levels too (in order) after sorting by specified level++        Returns+        -------+        sorted_obj : %(klass)s+        """++    @Appender(_shared_docs['sort_index'] % dict(axes="axes", klass="NDFrame"))+    def sort_index(self, axis=0, level=None, ascending=True, inplace=False,+                   kind='quicksort', na_position='last', sort_remaining=True):+        inplace = validate_bool_kwarg(inplace, 'inplace')+        axis = self._get_axis_number(axis)+        axis_name = self._get_axis_name(axis)+        labels = self._get_axis(axis)++        if level is not None:+            raise NotImplementedError("level is not implemented")+        if inplace:+            raise NotImplementedError("inplace is not implemented")++        sort_index = labels.argsort()+        if not ascending:+            sort_index = sort_index[::-1]++        new_axis = labels.take(sort_index)+        return self.reindex(**{axis_name: new_axis})++    _shared_docs['reindex'] = """+        Conform %(klass)s to new index with optional filling logic, placing+        NA/NaN in locations having no value in the previous index. A new object+        is produced unless the new index is equivalent to the current one and+        copy=False++        Parameters+        ----------+        %(optional_labels)s+        %(axes)s : array-like, optional (should be specified using keywords)+            New labels / index to conform to. Preferably an Index object to+            avoid duplicating data+        %(optional_axis)s+        method : {None, 'backfill'/'bfill', 'pad'/'ffill', 'nearest'}, optional+            method to use for filling holes in reindexed DataFrame.+            Please note: this is only applicable to DataFrames/Series with a+            monotonically increasing/decreasing index.++            * default: don't fill gaps+            * pad / ffill: propagate last valid observation forward to next+              valid+            * backfill / bfill: use next valid observation to fill gap+            * nearest: use nearest valid observations to fill gap++        copy : boolean, default True+            Return a new object, even if the passed indexes are the same+        level : int or name+            Broadcast across a level, matching Index values on the+            passed MultiIndex level+        fill_value : scalar, default np.NaN+            Value to use for missing values. Defaults to NaN, but can be any+            "compatible" value+        limit : int, default None+            Maximum number of consecutive elements to forward or backward fill+        tolerance : optional+            Maximum distance between original and new labels for inexact+            matches. The values of the index at the matching locations most+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.++            Tolerance may be a scalar value, which applies the same tolerance+            to all values, or list-like, which applies variable tolerance per+            element. List-like includes list, tuple, array, Series, and must be+            the same size as the index and its dtype must exactly match the+            index's type.++            .. versionadded:: 0.21.0 (list-like tolerance)++        Examples+        --------++        ``DataFrame.reindex`` supports two calling conventions++        * ``(index=index_labels, columns=column_labels, ...)``+        * ``(labels, axis={'index', 'columns'}, ...)``++        We *highly* recommend using keyword arguments to clarify your+        intent.++        Create a dataframe with some fictional data.++        >>> index = ['Firefox', 'Chrome', 'Safari', 'IE10', 'Konqueror']+        >>> df = pd.DataFrame({+        ...      'http_status': [200,200,404,404,301],+        ...      'response_time': [0.04, 0.02, 0.07, 0.08, 1.0]},+        ...       index=index)+        >>> df+                   http_status  response_time+        Firefox            200           0.04+        Chrome             200           0.02+        Safari             404           0.07+        IE10               404           0.08+        Konqueror          301           1.00++        Create a new index and reindex the dataframe. By default+        values in the new index that do not have corresponding+        records in the dataframe are assigned ``NaN``.++        >>> new_index= ['Safari', 'Iceweasel', 'Comodo Dragon', 'IE10',+        ...             'Chrome']+        >>> df.reindex(new_index)+                       http_status  response_time+        Safari               404.0           0.07+        Iceweasel              NaN            NaN+        Comodo Dragon          NaN            NaN+        IE10                 404.0           0.08+        Chrome               200.0           0.02++        We can fill in the missing values by passing a value to+        the keyword ``fill_value``. Because the index is not monotonically+        increasing or decreasing, we cannot use arguments to the keyword+        ``method`` to fill the ``NaN`` values.++        >>> df.reindex(new_index, fill_value=0)+                       http_status  response_time+        Safari                 404           0.07+        Iceweasel                0           0.00+        Comodo Dragon            0           0.00+        IE10                   404           0.08+        Chrome                 200           0.02++        >>> df.reindex(new_index, fill_value='missing')+                      http_status response_time+        Safari                404          0.07+        Iceweasel         missing       missing+        Comodo Dragon     missing       missing+        IE10                  404          0.08+        Chrome                200          0.02++        We can also reindex the columns.++        >>> df.reindex(columns=['http_status', 'user_agent'])+                   http_status  user_agent+        Firefox            200         NaN+        Chrome             200         NaN+        Safari             404         NaN+        IE10               404         NaN+        Konqueror          301         NaN++        Or we can use "axis-style" keyword arguments++        >>> df.reindex(['http_status', 'user_agent'], axis="columns")+                   http_status  user_agent+        Firefox            200         NaN+        Chrome             200         NaN+        Safari             404         NaN+        IE10               404         NaN+        Konqueror          301         NaN++        To further illustrate the filling functionality in+        ``reindex``, we will create a dataframe with a+        monotonically increasing index (for example, a sequence+        of dates).++        >>> date_index = pd.date_range('1/1/2010', periods=6, freq='D')+        >>> df2 = pd.DataFrame({"prices": [100, 101, np.nan, 100, 89, 88]},+        ...                    index=date_index)+        >>> df2+                    prices+        2010-01-01     100+        2010-01-02     101+        2010-01-03     NaN+        2010-01-04     100+        2010-01-05      89+        2010-01-06      88++        Suppose we decide to expand the dataframe to cover a wider+        date range.++        >>> date_index2 = pd.date_range('12/29/2009', periods=10, freq='D')+        >>> df2.reindex(date_index2)+                    prices+        2009-12-29     NaN+        2009-12-30     NaN+        2009-12-31     NaN+        2010-01-01     100+        2010-01-02     101+        2010-01-03     NaN+        2010-01-04     100+        2010-01-05      89+        2010-01-06      88+        2010-01-07     NaN++        The index entries that did not have a value in the original data frame+        (for example, '2009-12-29') are by default filled with ``NaN``.+        If desired, we can fill in the missing values using one of several+        options.++        For example, to back-propagate the last valid value to fill the ``NaN``+        values, pass ``bfill`` as an argument to the ``method`` keyword.++        >>> df2.reindex(date_index2, method='bfill')+                    prices+        2009-12-29     100+        2009-12-30     100+        2009-12-31     100+        2010-01-01     100+        2010-01-02     101+        2010-01-03     NaN+        2010-01-04     100+        2010-01-05      89+        2010-01-06      88+        2010-01-07     NaN++        Please note that the ``NaN`` value present in the original dataframe+        (at index value 2010-01-03) will not be filled by any of the+        value propagation schemes. This is because filling while reindexing+        does not look at dataframe values, but only compares the original and+        desired indexes. If you do want to fill in the ``NaN`` values present+        in the original dataframe, use the ``fillna()`` method.++        See the :ref:`user guide <basics.reindexing>` for more.++        Returns+        -------+        reindexed : %(klass)s+        """++    # TODO: Decide if we care about having different examples for different+    #       kinds++    @Appender(_shared_docs['reindex'] % dict(axes="axes", klass="NDFrame",+                                             optional_labels="",+                                             optional_axis=""))+    def reindex(self, *args, **kwargs):++        # construct the args+        axes, kwargs = self._construct_axes_from_arguments(args, kwargs)+        method = missing.clean_reindex_fill_method(kwargs.pop('method', None))+        level = kwargs.pop('level', None)+        copy = kwargs.pop('copy', True)+        limit = kwargs.pop('limit', None)+        tolerance = kwargs.pop('tolerance', None)+        fill_value = kwargs.pop('fill_value', None)++        # Series.reindex doesn't use / need the axis kwarg+        # We pop and ignore it here, to make writing Series/Frame generic code+        # easier+        kwargs.pop("axis", None)++        if kwargs:+            raise TypeError('reindex() got an unexpected keyword '+                            'argument "{0}"'.format(list(kwargs.keys())[0]))++        self._consolidate_inplace()++        # if all axes that are requested to reindex are equal, then only copy+        # if indicated must have index names equal here as well as values+        if all(self._get_axis(axis).identical(ax)+               for axis, ax in axes.items() if ax is not None):+            if copy:+                return self.copy()+            return self++        # check if we are a multi reindex+        if self._needs_reindex_multi(axes, method, level):+            try:+                return self._reindex_multi(axes, copy, fill_value)+            except Exception:+                pass++        # perform the reindex on the axes+        return self._reindex_axes(axes, level, limit, tolerance, method,+                                  fill_value, copy).__finalize__(self)++    def _reindex_axes(self, axes, level, limit, tolerance, method, fill_value,+                      copy):+        """Perform the reindex for all the axes."""+        obj = self+        for a in self._AXIS_ORDERS:+            labels = axes[a]+            if labels is None:+                continue++            ax = self._get_axis(a)+            new_index, indexer = ax.reindex(labels, level=level, limit=limit,+                                            tolerance=tolerance, method=method)++            axis = self._get_axis_number(a)+            obj = obj._reindex_with_indexers({axis: [new_index, indexer]},+                                             fill_value=fill_value,+                                             copy=copy, allow_dups=False)++        return obj++    def _needs_reindex_multi(self, axes, method, level):+        """Check if we do need a multi reindex."""+        return ((com.count_not_none(*axes.values()) == self._AXIS_LEN) and+                method is None and level is None and not self._is_mixed_type)++    def _reindex_multi(self, axes, copy, fill_value):+        return NotImplemented++    _shared_docs[+        'reindex_axis'] = ("""Conform input object to new index with optional+        filling logic, placing NA/NaN in locations having no value in the+        previous index. A new object is produced unless the new index is+        equivalent to the current one and copy=False++        Parameters+        ----------+        labels : array-like+            New labels / index to conform to. Preferably an Index object to+            avoid duplicating data+        axis : %(axes_single_arg)s+        method : {None, 'backfill'/'bfill', 'pad'/'ffill', 'nearest'}, optional+            Method to use for filling holes in reindexed DataFrame:++            * default: don't fill gaps+            * pad / ffill: propagate last valid observation forward to next+              valid+            * backfill / bfill: use next valid observation to fill gap+            * nearest: use nearest valid observations to fill gap++        copy : boolean, default True+            Return a new object, even if the passed indexes are the same+        level : int or name+            Broadcast across a level, matching Index values on the+            passed MultiIndex level+        limit : int, default None+            Maximum number of consecutive elements to forward or backward fill+        tolerance : optional+            Maximum distance between original and new labels for inexact+            matches. The values of the index at the matching locations most+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.++            Tolerance may be a scalar value, which applies the same tolerance+            to all values, or list-like, which applies variable tolerance per+            element. List-like includes list, tuple, array, Series, and must be+            the same size as the index and its dtype must exactly match the+            index's type.++            .. versionadded:: 0.21.0 (list-like tolerance)++        Examples+        --------+        >>> df.reindex_axis(['A', 'B', 'C'], axis=1)++        See Also+        --------+        reindex, reindex_like++        Returns+        -------+        reindexed : %(klass)s+        """)++    @Appender(_shared_docs['reindex_axis'] % _shared_doc_kwargs)+    def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True,+                     limit=None, fill_value=None):+        msg = ("'.reindex_axis' is deprecated and will be removed in a future "+               "version. Use '.reindex' instead.")+        self._consolidate_inplace()++        axis_name = self._get_axis_name(axis)+        axis_values = self._get_axis(axis_name)+        method = missing.clean_reindex_fill_method(method)+        warnings.warn(msg, FutureWarning, stacklevel=3)+        new_index, indexer = axis_values.reindex(labels, method, level,+                                                 limit=limit)+        return self._reindex_with_indexers({axis: [new_index, indexer]},+                                           fill_value=fill_value, copy=copy)++    def _reindex_with_indexers(self, reindexers, fill_value=None, copy=False,+                               allow_dups=False):+        """allow_dups indicates an internal call here """++        # reindex doing multiple operations on different axes if indicated+        new_data = self._data+        for axis in sorted(reindexers.keys()):+            index, indexer = reindexers[axis]+            baxis = self._get_block_manager_axis(axis)++            if index is None:+                continue++            index = ensure_index(index)+            if indexer is not None:+                indexer = ensure_int64(indexer)++            # TODO: speed up on homogeneous DataFrame objects+            new_data = new_data.reindex_indexer(index, indexer, axis=baxis,+                                                fill_value=fill_value,+                                                allow_dups=allow_dups,+                                                copy=copy)++        if copy and new_data is self._data:+            new_data = new_data.copy()++        return self._constructor(new_data).__finalize__(self)++    def _reindex_axis(self, new_index, fill_method, axis, copy):+        new_data = self._data.reindex_axis(new_index, axis=axis,+                                           method=fill_method, copy=copy)++        if new_data is self._data and not copy:+            return self+        else:+            return self._constructor(new_data).__finalize__(self)++    def filter(self, items=None, like=None, regex=None, axis=None):+        """+        Subset rows or columns of dataframe according to labels in+        the specified index.++        Note that this routine does not filter a dataframe on its+        contents. The filter is applied to the labels of the index.++        Parameters+        ----------+        items : list-like+            List of axis to restrict to (must not all be present).+        like : string+            Keep axis where "arg in col == True".+        regex : string (regular expression)+            Keep axis with re.search(regex, col) == True.+        axis : int or string axis name+            The axis to filter on.  By default this is the info axis,+            'index' for Series, 'columns' for DataFrame.++        Returns+        -------+        same type as input object++        Examples+        --------+        >>> df = pd.DataFrame(np.array(([1,2,3], [4,5,6])),+        ...                   index=['mouse', 'rabbit'],+        ...                   columns=['one', 'two', 'three'])++        >>> # select columns by name+        >>> df.filter(items=['one', 'three'])+                 one  three+        mouse     1      3+        rabbit    4      6++        >>> # select columns by regular expression+        >>> df.filter(regex='e$', axis=1)+                 one  three+        mouse     1      3+        rabbit    4      6++        >>> # select rows containing 'bbi'+        >>> df.filter(like='bbi', axis=0)+                 one  two  three+        rabbit    4    5      6++        See Also+        --------+        pandas.DataFrame.loc++        Notes+        -----+        The ``items``, ``like``, and ``regex`` parameters are+        enforced to be mutually exclusive.++        ``axis`` defaults to the info axis that is used when indexing+        with ``[]``.+        """+        import re++        nkw = com.count_not_none(items, like, regex)+        if nkw > 1:+            raise TypeError('Keyword arguments `items`, `like`, or `regex` '+                            'are mutually exclusive')++        if axis is None:+            axis = self._info_axis_name+        labels = self._get_axis(axis)++        if items is not None:+            name = self._get_axis_name(axis)+            return self.reindex(+                **{name: [r for r in items if r in labels]})+        elif like:+            def f(x):+                return like in to_str(x)+            values = labels.map(f)+            return self.loc(axis=axis)[values]+        elif regex:+            def f(x):+                return matcher.search(to_str(x)) is not None+            matcher = re.compile(regex)+            values = labels.map(f)+            return self.loc(axis=axis)[values]+        else:+            raise TypeError('Must pass either `items`, `like`, or `regex`')++    def head(self, n=5):+        """+        Return the first `n` rows.++        This function returns the first `n` rows for the object based+        on position. It is useful for quickly testing if your object+        has the right type of data in it.++        Parameters+        ----------+        n : int, default 5+            Number of rows to select.++        Returns+        -------+        obj_head : same type as caller+            The first `n` rows of the caller object.++        See Also+        --------+        pandas.DataFrame.tail: Returns the last `n` rows.++        Examples+        --------+        >>> df = pd.DataFrame({'animal':['alligator', 'bee', 'falcon', 'lion',+        ...                    'monkey', 'parrot', 'shark', 'whale', 'zebra']})+        >>> df+              animal+        0  alligator+        1        bee+        2     falcon+        3       lion+        4     monkey+        5     parrot+        6      shark+        7      whale+        8      zebra++        Viewing the first 5 lines++        >>> df.head()+              animal+        0  alligator+        1        bee+        2     falcon+        3       lion+        4     monkey++        Viewing the first `n` lines (three in this case)++        >>> df.head(3)+              animal+        0  alligator+        1        bee+        2     falcon+        """++        return self.iloc[:n]++    def tail(self, n=5):+        """+        Return the last `n` rows.++        This function returns last `n` rows from the object based on+        position. It is useful for quickly verifying data, for example,+        after sorting or appending rows.++        Parameters+        ----------+        n : int, default 5+            Number of rows to select.++        Returns+        -------+        type of caller+            The last `n` rows of the caller object.++        See Also+        --------+        pandas.DataFrame.head : The first `n` rows of the caller object.++        Examples+        --------+        >>> df = pd.DataFrame({'animal':['alligator', 'bee', 'falcon', 'lion',+        ...                    'monkey', 'parrot', 'shark', 'whale', 'zebra']})+        >>> df+              animal+        0  alligator+        1        bee+        2     falcon+        3       lion+        4     monkey+        5     parrot+        6      shark+        7      whale+        8      zebra++        Viewing the last 5 lines++        >>> df.tail()+           animal+        4  monkey+        5  parrot+        6   shark+        7   whale+        8   zebra++        Viewing the last `n` lines (three in this case)++        >>> df.tail(3)+          animal+        6  shark+        7  whale+        8  zebra+        """++        if n == 0:+            return self.iloc[0:0]+        return self.iloc[-n:]++    def sample(self, n=None, frac=None, replace=False, weights=None,+               random_state=None, axis=None):+        """+        Return a random sample of items from an axis of object.++        You can use `random_state` for reproducibility.++        Parameters+        ----------+        n : int, optional+            Number of items from axis to return. Cannot be used with `frac`.+            Default = 1 if `frac` = None.+        frac : float, optional+            Fraction of axis items to return. Cannot be used with `n`.+        replace : boolean, optional+            Sample with or without replacement. Default = False.+        weights : str or ndarray-like, optional+            Default 'None' results in equal probability weighting.+            If passed a Series, will align with target object on index. Index+            values in weights not found in sampled object will be ignored and+            index values in sampled object not in weights will be assigned+            weights of zero.+            If called on a DataFrame, will accept the name of a column+            when axis = 0.+            Unless weights are a Series, weights must be same length as axis+            being sampled.+            If weights do not sum to 1, they will be normalized to sum to 1.+            Missing values in the weights column will be treated as zero.+            inf and -inf values not allowed.+        random_state : int or numpy.random.RandomState, optional+            Seed for the random number generator (if int), or numpy RandomState+            object.+        axis : int or string, optional+            Axis to sample. Accepts axis number or name. Default is stat axis+            for given data type (0 for Series and DataFrames, 1 for Panels).++        Returns+        -------+        A new object of same type as caller.++        Examples+        --------+        Generate an example ``Series`` and ``DataFrame``:++        >>> s = pd.Series(np.random.randn(50))+        >>> s.head()+        0   -0.038497+        1    1.820773+        2   -0.972766+        3   -1.598270+        4   -1.095526+        dtype: float64+        >>> df = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD'))+        >>> df.head()+                  A         B         C         D+        0  0.016443 -2.318952 -0.566372 -1.028078+        1 -1.051921  0.438836  0.658280 -0.175797+        2 -1.243569 -0.364626 -0.215065  0.057736+        3  1.768216  0.404512 -0.385604 -1.457834+        4  1.072446 -1.137172  0.314194 -0.046661++        Next extract a random sample from both of these objects...++        3 random elements from the ``Series``:++        >>> s.sample(n=3)+        27   -0.994689+        55   -1.049016+        67   -0.224565+        dtype: float64++        And a random 10% of the ``DataFrame`` with replacement:++        >>> df.sample(frac=0.1, replace=True)+                   A         B         C         D+        35  1.981780  0.142106  1.817165 -0.290805+        49 -1.336199 -0.448634 -0.789640  0.217116+        40  0.823173 -0.078816  1.009536  1.015108+        15  1.421154 -0.055301 -1.922594 -0.019696+        6  -0.148339  0.832938  1.787600 -1.383767++        You can use `random state` for reproducibility:++        >>> df.sample(random_state=1)+        A         B         C         D+        37 -2.027662  0.103611  0.237496 -0.165867+        43 -0.259323 -0.583426  1.516140 -0.479118+        12 -1.686325 -0.579510  0.985195 -0.460286+        8   1.167946  0.429082  1.215742 -1.636041+        9   1.197475 -0.864188  1.554031 -1.505264+        """++        if axis is None:+            axis = self._stat_axis_number++        axis = self._get_axis_number(axis)+        axis_length = self.shape[axis]++        # Process random_state argument+        rs = com.random_state(random_state)++        # Check weights for compliance+        if weights is not None:++            # If a series, align with frame+            if isinstance(weights, pd.Series):+                weights = weights.reindex(self.axes[axis])++            # Strings acceptable if a dataframe and axis = 0+            if isinstance(weights, string_types):+                if isinstance(self, pd.DataFrame):+                    if axis == 0:+                        try:+                            weights = self[weights]+                        except KeyError:+                            raise KeyError("String passed to weights not a "+                                           "valid column")+                    else:+                        raise ValueError("Strings can only be passed to "+                                         "weights when sampling from rows on "+                                         "a DataFrame")+                else:+                    raise ValueError("Strings cannot be passed as weights "+                                     "when sampling from a Series or Panel.")++            weights = pd.Series(weights, dtype='float64')++            if len(weights) != axis_length:+                raise ValueError("Weights and axis to be sampled must be of "+                                 "same length")++            if (weights == np.inf).any() or (weights == -np.inf).any():+                raise ValueError("weight vector may not include `inf` values")++            if (weights < 0).any():+                raise ValueError("weight vector many not include negative "+                                 "values")++            # If has nan, set to zero.+            weights = weights.fillna(0)++            # Renormalize if don't sum to 1+            if weights.sum() != 1:+                if weights.sum() != 0:+                    weights = weights / weights.sum()+                else:+                    raise ValueError("Invalid weights: weights sum to zero")++            weights = weights.values++        # If no frac or n, default to n=1.+        if n is None and frac is None:+            n = 1+        elif n is not None and frac is None and n % 1 != 0:+            raise ValueError("Only integers accepted as `n` values")+        elif n is None and frac is not None:+            n = int(round(frac * axis_length))+        elif n is not None and frac is not None:+            raise ValueError('Please enter a value for `frac` OR `n`, not '+                             'both')++        # Check for negative sizes+        if n < 0:+            raise ValueError("A negative number of rows requested. Please "+                             "provide positive value.")++        locs = rs.choice(axis_length, size=n, replace=replace, p=weights)+        return self.take(locs, axis=axis, is_copy=False)++    _shared_docs['pipe'] = (r"""+        Apply func(self, \*args, \*\*kwargs)++        Parameters+        ----------+        func : function+            function to apply to the %(klass)s.+            ``args``, and ``kwargs`` are passed into ``func``.+            Alternatively a ``(callable, data_keyword)`` tuple where+            ``data_keyword`` is a string indicating the keyword of+            ``callable`` that expects the %(klass)s.+        args : iterable, optional+            positional arguments passed into ``func``.+        kwargs : mapping, optional+            a dictionary of keyword arguments passed into ``func``.++        Returns+        -------+        object : the return type of ``func``.++        Notes+        -----++        Use ``.pipe`` when chaining together functions that expect+        Series, DataFrames or GroupBy objects. Instead of writing++        >>> f(g(h(df), arg1=a), arg2=b, arg3=c)++        You can write++        >>> (df.pipe(h)+        ...    .pipe(g, arg1=a)+        ...    .pipe(f, arg2=b, arg3=c)+        ... )++        If you have a function that takes the data as (say) the second+        argument, pass a tuple indicating which keyword expects the+        data. For example, suppose ``f`` takes its data as ``arg2``:++        >>> (df.pipe(h)+        ...    .pipe(g, arg1=a)+        ...    .pipe((f, 'arg2'), arg1=a, arg3=c)+        ...  )++        See Also+        --------+        pandas.DataFrame.apply+        pandas.DataFrame.applymap+        pandas.Series.map+    """)++    @Appender(_shared_docs['pipe'] % _shared_doc_kwargs)+    def pipe(self, func, *args, **kwargs):+        return com._pipe(self, func, *args, **kwargs)++    _shared_docs['aggregate'] = ("""+    Aggregate using one or more operations over the specified axis.++    %(versionadded)s++    Parameters+    ----------+    func : function, string, dictionary, or list of string/functions+        Function to use for aggregating the data. If a function, must either+        work when passed a %(klass)s or when passed to %(klass)s.apply. For+        a DataFrame, can pass a dict, if the keys are DataFrame column names.++        Accepted combinations are:++        - string function name.+        - function.+        - list of functions.+        - dict of column names -> functions (or list of functions).++    %(axis)s+    *args+        Positional arguments to pass to `func`.+    **kwargs+        Keyword arguments to pass to `func`.++    Returns+    -------+    aggregated : %(klass)s++    Notes+    -----+    `agg` is an alias for `aggregate`. Use the alias.++    A passed user-defined-function will be passed a Series for evaluation.+    """)++    _shared_docs['transform'] = ("""+    Call function producing a like-indexed %(klass)s+    and return a %(klass)s with the transformed values++    .. versionadded:: 0.20.0++    Parameters+    ----------+    func : callable, string, dictionary, or list of string/callables+        To apply to column++        Accepted Combinations are:++        - string function name+        - function+        - list of functions+        - dict of column names -> functions (or list of functions)++    Returns+    -------+    transformed : %(klass)s++    Examples+    --------+    >>> df = pd.DataFrame(np.random.randn(10, 3), columns=['A', 'B', 'C'],+    ...                   index=pd.date_range('1/1/2000', periods=10))+    df.iloc[3:7] = np.nan++    >>> df.transform(lambda x: (x - x.mean()) / x.std())+                       A         B         C+    2000-01-01  0.579457  1.236184  0.123424+    2000-01-02  0.370357 -0.605875 -1.231325+    2000-01-03  1.455756 -0.277446  0.288967+    2000-01-04       NaN       NaN       NaN+    2000-01-05       NaN       NaN       NaN+    2000-01-06       NaN       NaN       NaN+    2000-01-07       NaN       NaN       NaN+    2000-01-08 -0.498658  1.274522  1.642524+    2000-01-09 -0.540524 -1.012676 -0.828968+    2000-01-10 -1.366388 -0.614710  0.005378++    See also+    --------+    pandas.%(klass)s.aggregate+    pandas.%(klass)s.apply+    """)++    # ----------------------------------------------------------------------+    # Attribute access++    def __finalize__(self, other, method=None, **kwargs):+        """+        Propagate metadata from other to self.++        Parameters+        ----------+        other : the object from which to get the attributes that we are going+            to propagate+        method : optional, a passed method name ; possibly to take different+            types of propagation actions based on this++        """+        if isinstance(other, NDFrame):+            for name in self._metadata:+                object.__setattr__(self, name, getattr(other, name, None))+        return self++    def __getattr__(self, name):+        """After regular attribute access, try looking up the name+        This allows simpler access to columns for interactive use.+        """++        # Note: obj.x will always call obj.__getattribute__('x') prior to+        # calling obj.__getattr__('x').++        if (name in self._internal_names_set or name in self._metadata or+                name in self._accessors):+            return object.__getattribute__(self, name)+        else:+            if self._info_axis._can_hold_identifiers_and_holds_name(name):+                return self[name]+            return object.__getattribute__(self, name)++    def __setattr__(self, name, value):+        """After regular attribute access, try setting the name+        This allows simpler access to columns for interactive use.+        """++        # first try regular attribute access via __getattribute__, so that+        # e.g. ``obj.x`` and ``obj.x = 4`` will always reference/modify+        # the same attribute.++        try:+            object.__getattribute__(self, name)+            return object.__setattr__(self, name, value)+        except AttributeError:+            pass++        # if this fails, go on to more involved attribute setting+        # (note that this matches __getattr__, above).+        if name in self._internal_names_set:+            object.__setattr__(self, name, value)+        elif name in self._metadata:+            object.__setattr__(self, name, value)+        else:+            try:+                existing = getattr(self, name)+                if isinstance(existing, Index):+                    object.__setattr__(self, name, value)+                elif name in self._info_axis:+                    self[name] = value+                else:+                    object.__setattr__(self, name, value)+            except (AttributeError, TypeError):+                if isinstance(self, ABCDataFrame) and (is_list_like(value)):+                    warnings.warn("Pandas doesn't allow columns to be "+                                  "created via a new attribute name - see "+                                  "https://pandas.pydata.org/pandas-docs/"+                                  "stable/indexing.html#attribute-access",+                                  stacklevel=2)+                object.__setattr__(self, name, value)++    # ----------------------------------------------------------------------+    # Getting and setting elements++    # ----------------------------------------------------------------------+    # Consolidation of internals++    def _protect_consolidate(self, f):+        """Consolidate _data -- if the blocks have changed, then clear the+        cache+        """+        blocks_before = len(self._data.blocks)+        result = f()+        if len(self._data.blocks) != blocks_before:+            self._clear_item_cache()+        return result++    def _consolidate_inplace(self):+        """Consolidate data in place and return None"""++        def f():+            self._data = self._data.consolidate()++        self._protect_consolidate(f)++    def _consolidate(self, inplace=False):+        """+        Compute NDFrame with "consolidated" internals (data of each dtype+        grouped together in a single ndarray).++        Parameters+        ----------+        inplace : boolean, default False+            If False return new object, otherwise modify existing object++        Returns+        -------+        consolidated : same type as caller+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        if inplace:+            self._consolidate_inplace()+        else:+            f = lambda: self._data.consolidate()+            cons_data = self._protect_consolidate(f)+            return self._constructor(cons_data).__finalize__(self)++    def consolidate(self, inplace=False):+        """Compute NDFrame with "consolidated" internals (data of each dtype+        grouped together in a single ndarray).++        .. deprecated:: 0.20.0+            Consolidate will be an internal implementation only.+        """+        # 15483+        warnings.warn("consolidate is deprecated and will be removed in a "+                      "future release.", FutureWarning, stacklevel=2)+        return self._consolidate(inplace)++    @property+    def _is_mixed_type(self):+        f = lambda: self._data.is_mixed_type+        return self._protect_consolidate(f)++    @property+    def _is_numeric_mixed_type(self):+        f = lambda: self._data.is_numeric_mixed_type+        return self._protect_consolidate(f)++    @property+    def _is_datelike_mixed_type(self):+        f = lambda: self._data.is_datelike_mixed_type+        return self._protect_consolidate(f)++    def _check_inplace_setting(self, value):+        """ check whether we allow in-place setting with this type of value """++        if self._is_mixed_type:+            if not self._is_numeric_mixed_type:++                # allow an actual np.nan thru+                try:+                    if np.isnan(value):+                        return True+                except Exception:+                    pass++                raise TypeError('Cannot do inplace boolean setting on '+                                'mixed-types with a non np.nan value')++        return True++    def _get_numeric_data(self):+        return self._constructor(+            self._data.get_numeric_data()).__finalize__(self)++    def _get_bool_data(self):+        return self._constructor(self._data.get_bool_data()).__finalize__(self)++    # ----------------------------------------------------------------------+    # Internal Interface Methods++    def as_matrix(self, columns=None):+        """Convert the frame to its Numpy-array representation.++        .. deprecated:: 0.23.0+            Use :meth:`DataFrame.values` instead.++        Parameters+        ----------+        columns: list, optional, default:None+            If None, return all columns, otherwise, returns specified columns.++        Returns+        -------+        values : ndarray+            If the caller is heterogeneous and contains booleans or objects,+            the result will be of dtype=object. See Notes.+++        Notes+        -----+        Return is NOT a Numpy-matrix, rather, a Numpy-array.++        The dtype will be a lower-common-denominator dtype (implicit+        upcasting); that is to say if the dtypes (even of numeric types)+        are mixed, the one that accommodates all will be chosen. Use this+        with care if you are not dealing with the blocks.++        e.g. If the dtypes are float16 and float32, dtype will be upcast to+        float32.  If dtypes are int32 and uint8, dtype will be upcase to+        int32. By numpy.find_common_type convention, mixing int64 and uint64+        will result in a float64 dtype.++        This method is provided for backwards compatibility. Generally,+        it is recommended to use '.values'.++        See Also+        --------+        pandas.DataFrame.values+        """+        warnings.warn("Method .as_matrix will be removed in a future version. "+                      "Use .values instead.", FutureWarning, stacklevel=2)+        self._consolidate_inplace()+        return self._data.as_array(transpose=self._AXIS_REVERSED,+                                   items=columns)++    @property+    def values(self):+        """+        Return a Numpy representation of the DataFrame.++        Only the values in the DataFrame will be returned, the axes labels+        will be removed.++        Returns+        -------+        numpy.ndarray+            The values of the DataFrame.++        Examples+        --------+        A DataFrame where all columns are the same type (e.g., int64) results+        in an array of the same type.++        >>> df = pd.DataFrame({'age':    [ 3,  29],+        ...                    'height': [94, 170],+        ...                    'weight': [31, 115]})+        >>> df+           age  height  weight+        0    3      94      31+        1   29     170     115+        >>> df.dtypes+        age       int64+        height    int64+        weight    int64+        dtype: object+        >>> df.values+        array([[  3,  94,  31],+               [ 29, 170, 115]], dtype=int64)++        A DataFrame with mixed type columns(e.g., str/object, int64, float32)+        results in an ndarray of the broadest type that accommodates these+        mixed types (e.g., object).++        >>> df2 = pd.DataFrame([('parrot',   24.0, 'second'),+        ...                     ('lion',     80.5, 1),+        ...                     ('monkey', np.nan, None)],+        ...                   columns=('name', 'max_speed', 'rank'))+        >>> df2.dtypes+        name          object+        max_speed    float64+        rank          object+        dtype: object+        >>> df2.values+        array([['parrot', 24.0, 'second'],+               ['lion', 80.5, 1],+               ['monkey', nan, None]], dtype=object)++        Notes+        -----+        The dtype will be a lower-common-denominator dtype (implicit+        upcasting); that is to say if the dtypes (even of numeric types)+        are mixed, the one that accommodates all will be chosen. Use this+        with care if you are not dealing with the blocks.++        e.g. If the dtypes are float16 and float32, dtype will be upcast to+        float32.  If dtypes are int32 and uint8, dtype will be upcast to+        int32. By :func:`numpy.find_common_type` convention, mixing int64+        and uint64 will result in a float64 dtype.++        See Also+        --------+        pandas.DataFrame.index : Retrieve the index labels+        pandas.DataFrame.columns : Retrieving the column names+        """+        self._consolidate_inplace()+        return self._data.as_array(transpose=self._AXIS_REVERSED)++    @property+    def _values(self):+        """internal implementation"""+        return self.values++    @property+    def _get_values(self):+        # compat+        return self.values++    def get_values(self):+        """+        Return an ndarray after converting sparse values to dense.++        This is the same as ``.values`` for non-sparse data. For sparse+        data contained in a `pandas.SparseArray`, the data are first+        converted to a dense representation.++        Returns+        -------+        numpy.ndarray+            Numpy representation of DataFrame++        See Also+        --------+        values : Numpy representation of DataFrame.+        pandas.SparseArray : Container for sparse data.++        Examples+        --------+        >>> df = pd.DataFrame({'a': [1, 2], 'b': [True, False],+        ...                    'c': [1.0, 2.0]})+        >>> df+           a      b    c+        0  1   True  1.0+        1  2  False  2.0++        >>> df.get_values()+        array([[1, True, 1.0], [2, False, 2.0]], dtype=object)++        >>> df = pd.DataFrame({"a": pd.SparseArray([1, None, None]),+        ...                    "c": [1.0, 2.0, 3.0]})+        >>> df+             a    c+        0  1.0  1.0+        1  NaN  2.0+        2  NaN  3.0++        >>> df.get_values()+        array([[ 1.,  1.],+               [nan,  2.],+               [nan,  3.]])+        """+        return self.values++    def get_dtype_counts(self):+        """+        Return counts of unique dtypes in this object.++        Returns+        -------+        dtype : Series+            Series with the count of columns with each dtype.++        See Also+        --------+        dtypes : Return the dtypes in this object.++        Examples+        --------+        >>> a = [['a', 1, 1.0], ['b', 2, 2.0], ['c', 3, 3.0]]+        >>> df = pd.DataFrame(a, columns=['str', 'int', 'float'])+        >>> df+          str  int  float+        0   a    1    1.0+        1   b    2    2.0+        2   c    3    3.0++        >>> df.get_dtype_counts()+        float64    1+        int64      1+        object     1+        dtype: int64+        """+        from pandas import Series+        return Series(self._data.get_dtype_counts())++    def get_ftype_counts(self):+        """+        Return counts of unique ftypes in this object.++        .. deprecated:: 0.23.0++        This is useful for SparseDataFrame or for DataFrames containing+        sparse arrays.++        Returns+        -------+        dtype : Series+            Series with the count of columns with each type and+            sparsity (dense/sparse)++        See Also+        --------+        ftypes : Return ftypes (indication of sparse/dense and dtype) in+            this object.++        Examples+        --------+        >>> a = [['a', 1, 1.0], ['b', 2, 2.0], ['c', 3, 3.0]]+        >>> df = pd.DataFrame(a, columns=['str', 'int', 'float'])+        >>> df+          str  int  float+        0   a    1    1.0+        1   b    2    2.0+        2   c    3    3.0++        >>> df.get_ftype_counts()+        float64:dense    1+        int64:dense      1+        object:dense     1+        dtype: int64+        """+        warnings.warn("get_ftype_counts is deprecated and will "+                      "be removed in a future version",+                      FutureWarning, stacklevel=2)++        from pandas import Series+        return Series(self._data.get_ftype_counts())++    @property+    def dtypes(self):+        """+        Return the dtypes in the DataFrame.++        This returns a Series with the data type of each column.+        The result's index is the original DataFrame's columns. Columns+        with mixed types are stored with the ``object`` dtype. See+        :ref:`the User Guide <basics.dtypes>` for more.++        Returns+        -------+        pandas.Series+            The data type of each column.++        See Also+        --------+        pandas.DataFrame.ftypes : dtype and sparsity information.++        Examples+        --------+        >>> df = pd.DataFrame({'float': [1.0],+        ...                    'int': [1],+        ...                    'datetime': [pd.Timestamp('20180310')],+        ...                    'string': ['foo']})+        >>> df.dtypes+        float              float64+        int                  int64+        datetime    datetime64[ns]+        string              object+        dtype: object+        """+        from pandas import Series+        return Series(self._data.get_dtypes(), index=self._info_axis,+                      dtype=np.object_)++    @property+    def ftypes(self):+        """+        Return the ftypes (indication of sparse/dense and dtype) in DataFrame.++        This returns a Series with the data type of each column.+        The result's index is the original DataFrame's columns. Columns+        with mixed types are stored with the ``object`` dtype.  See+        :ref:`the User Guide <basics.dtypes>` for more.++        Returns+        -------+        pandas.Series+            The data type and indication of sparse/dense of each column.++        See Also+        --------+        pandas.DataFrame.dtypes: Series with just dtype information.+        pandas.SparseDataFrame : Container for sparse tabular data.++        Notes+        -----+        Sparse data should have the same dtypes as its dense representation.++        Examples+        --------+        >>> arr = np.random.RandomState(0).randn(100, 4)+        >>> arr[arr < .8] = np.nan+        >>> pd.DataFrame(arr).ftypes+        0    float64:dense+        1    float64:dense+        2    float64:dense+        3    float64:dense+        dtype: object++        >>> pd.SparseDataFrame(arr).ftypes+        0    float64:sparse+        1    float64:sparse+        2    float64:sparse+        3    float64:sparse+        dtype: object+        """+        from pandas import Series+        return Series(self._data.get_ftypes(), index=self._info_axis,+                      dtype=np.object_)++    def as_blocks(self, copy=True):+        """+        Convert the frame to a dict of dtype -> Constructor Types that each has+        a homogeneous dtype.++        .. deprecated:: 0.21.0++        NOTE: the dtypes of the blocks WILL BE PRESERVED HERE (unlike in+              as_matrix)++        Parameters+        ----------+        copy : boolean, default True++        Returns+        -------+        values : a dict of dtype -> Constructor Types+        """+        warnings.warn("as_blocks is deprecated and will "+                      "be removed in a future version",+                      FutureWarning, stacklevel=2)+        return self._to_dict_of_blocks(copy=copy)++    @property+    def blocks(self):+        """+        Internal property, property synonym for as_blocks()++        .. deprecated:: 0.21.0+        """+        return self.as_blocks()++    def _to_dict_of_blocks(self, copy=True):+        """+        Return a dict of dtype -> Constructor Types that+        each is a homogeneous dtype.++        Internal ONLY+        """+        return {k: self._constructor(v).__finalize__(self)+                for k, v, in self._data.to_dict(copy=copy).items()}++    @deprecate_kwarg(old_arg_name='raise_on_error', new_arg_name='errors',+                     mapping={True: 'raise', False: 'ignore'})+    def astype(self, dtype, copy=True, errors='raise', **kwargs):+        """+        Cast a pandas object to a specified dtype ``dtype``.++        Parameters+        ----------+        dtype : data type, or dict of column name -> data type+            Use a numpy.dtype or Python type to cast entire pandas object to+            the same type. Alternatively, use {col: dtype, ...}, where col is a+            column label and dtype is a numpy.dtype or Python type to cast one+            or more of the DataFrame's columns to column-specific types.+        copy : bool, default True.+            Return a copy when ``copy=True`` (be very careful setting+            ``copy=False`` as changes to values then may propagate to other+            pandas objects).+        errors : {'raise', 'ignore'}, default 'raise'.+            Control raising of exceptions on invalid data for provided dtype.++            - ``raise`` : allow exceptions to be raised+            - ``ignore`` : suppress exceptions. On error return original object++            .. versionadded:: 0.20.0++        raise_on_error : raise on invalid input+            .. deprecated:: 0.20.0+               Use ``errors`` instead+        kwargs : keyword arguments to pass on to the constructor++        Returns+        -------+        casted : same type as caller++        Examples+        --------+        >>> ser = pd.Series([1, 2], dtype='int32')+        >>> ser+        0    1+        1    2+        dtype: int32+        >>> ser.astype('int64')+        0    1+        1    2+        dtype: int64++        Convert to categorical type:++        >>> ser.astype('category')+        0    1+        1    2+        dtype: category+        Categories (2, int64): [1, 2]++        Convert to ordered categorical type with custom ordering:++        >>> ser.astype('category', ordered=True, categories=[2, 1])+        0    1+        1    2+        dtype: category+        Categories (2, int64): [2 < 1]++        Note that using ``copy=False`` and changing data on a new+        pandas object may propagate changes:++        >>> s1 = pd.Series([1,2])+        >>> s2 = s1.astype('int64', copy=False)+        >>> s2[0] = 10+        >>> s1  # note that s1[0] has changed too+        0    10+        1     2+        dtype: int64++        See also+        --------+        pandas.to_datetime : Convert argument to datetime.+        pandas.to_timedelta : Convert argument to timedelta.+        pandas.to_numeric : Convert argument to a numeric type.+        numpy.ndarray.astype : Cast a numpy array to a specified type.+        """+        if is_dict_like(dtype):+            if self.ndim == 1:  # i.e. Series+                if len(dtype) > 1 or self.name not in dtype:+                    raise KeyError('Only the Series name can be used for '+                                   'the key in Series dtype mappings.')+                new_type = dtype[self.name]+                return self.astype(new_type, copy, errors, **kwargs)+            elif self.ndim > 2:+                raise NotImplementedError(+                    'astype() only accepts a dtype arg of type dict when '+                    'invoked on Series and DataFrames. A single dtype must be '+                    'specified when invoked on a Panel.'+                )+            for col_name in dtype.keys():+                if col_name not in self:+                    raise KeyError('Only a column name can be used for the '+                                   'key in a dtype mappings argument.')+            results = []+            for col_name, col in self.iteritems():+                if col_name in dtype:+                    results.append(col.astype(dtype[col_name], copy=copy))+                else:+                    results.append(results.append(col.copy() if copy else col))++        elif is_categorical_dtype(dtype) and self.ndim > 1:+            # GH 18099: columnwise conversion to categorical+            results = (self[col].astype(dtype, copy=copy) for col in self)++        else:+            # else, only a single dtype is given+            new_data = self._data.astype(dtype=dtype, copy=copy, errors=errors,+                                         **kwargs)+            return self._constructor(new_data).__finalize__(self)++        # GH 19920: retain column metadata after concat+        result = pd.concat(results, axis=1, copy=False)+        result.columns = self.columns+        return result++    def copy(self, deep=True):+        """+        Make a copy of this object's indices and data.++        When ``deep=True`` (default), a new object will be created with a+        copy of the calling object's data and indices. Modifications to+        the data or indices of the copy will not be reflected in the+        original object (see notes below).++        When ``deep=False``, a new object will be created without copying+        the calling object's data or index (only references to the data+        and index are copied). Any changes to the data of the original+        will be reflected in the shallow copy (and vice versa).++        Parameters+        ----------+        deep : bool, default True+            Make a deep copy, including a copy of the data and the indices.+            With ``deep=False`` neither the indices nor the data are copied.++        Returns+        -------+        copy : Series, DataFrame or Panel+            Object type matches caller.++        Notes+        -----+        When ``deep=True``, data is copied but actual Python objects+        will not be copied recursively, only the reference to the object.+        This is in contrast to `copy.deepcopy` in the Standard Library,+        which recursively copies object data (see examples below).++        While ``Index`` objects are copied when ``deep=True``, the underlying+        numpy array is not copied for performance reasons. Since ``Index`` is+        immutable, the underlying data can be safely shared and a copy+        is not needed.++        Examples+        --------+        >>> s = pd.Series([1, 2], index=["a", "b"])+        >>> s+        a    1+        b    2+        dtype: int64++        >>> s_copy = s.copy()+        >>> s_copy+        a    1+        b    2+        dtype: int64++        **Shallow copy versus default (deep) copy:**++        >>> s = pd.Series([1, 2], index=["a", "b"])+        >>> deep = s.copy()+        >>> shallow = s.copy(deep=False)++        Shallow copy shares data and index with original.++        >>> s is shallow+        False+        >>> s.values is shallow.values and s.index is shallow.index+        True++        Deep copy has own copy of data and index.++        >>> s is deep+        False+        >>> s.values is deep.values or s.index is deep.index+        False++        Updates to the data shared by shallow copy and original is reflected+        in both; deep copy remains unchanged.++        >>> s[0] = 3+        >>> shallow[1] = 4+        >>> s+        a    3+        b    4+        dtype: int64+        >>> shallow+        a    3+        b    4+        dtype: int64+        >>> deep+        a    1+        b    2+        dtype: int64++        Note that when copying an object containing Python objects, a deep copy+        will copy the data, but will not do so recursively. Updating a nested+        data object will be reflected in the deep copy.++        >>> s = pd.Series([[1, 2], [3, 4]])+        >>> deep = s.copy()+        >>> s[0][0] = 10+        >>> s+        0    [10, 2]+        1     [3, 4]+        dtype: object+        >>> deep+        0    [10, 2]+        1     [3, 4]+        dtype: object+        """+        data = self._data.copy(deep=deep)+        return self._constructor(data).__finalize__(self)++    def __copy__(self, deep=True):+        return self.copy(deep=deep)++    def __deepcopy__(self, memo=None):+        """+        Parameters+        ----------+        memo, default None+            Standard signature. Unused+        """+        if memo is None:+            memo = {}+        return self.copy(deep=True)++    def _convert(self, datetime=False, numeric=False, timedelta=False,+                 coerce=False, copy=True):+        """+        Attempt to infer better dtype for object columns++        Parameters+        ----------+        datetime : boolean, default False+            If True, convert to date where possible.+        numeric : boolean, default False+            If True, attempt to convert to numbers (including strings), with+            unconvertible values becoming NaN.+        timedelta : boolean, default False+            If True, convert to timedelta where possible.+        coerce : boolean, default False+            If True, force conversion with unconvertible values converted to+            nulls (NaN or NaT)+        copy : boolean, default True+            If True, return a copy even if no copy is necessary (e.g. no+            conversion was done). Note: This is meant for internal use, and+            should not be confused with inplace.++        Returns+        -------+        converted : same as input object+        """+        return self._constructor(+            self._data.convert(datetime=datetime, numeric=numeric,+                               timedelta=timedelta, coerce=coerce,+                               copy=copy)).__finalize__(self)++    def convert_objects(self, convert_dates=True, convert_numeric=False,+                        convert_timedeltas=True, copy=True):+        """Attempt to infer better dtype for object columns.++        .. deprecated:: 0.21.0++        Parameters+        ----------+        convert_dates : boolean, default True+            If True, convert to date where possible. If 'coerce', force+            conversion, with unconvertible values becoming NaT.+        convert_numeric : boolean, default False+            If True, attempt to coerce to numbers (including strings), with+            unconvertible values becoming NaN.+        convert_timedeltas : boolean, default True+            If True, convert to timedelta where possible. If 'coerce', force+            conversion, with unconvertible values becoming NaT.+        copy : boolean, default True+            If True, return a copy even if no copy is necessary (e.g. no+            conversion was done). Note: This is meant for internal use, and+            should not be confused with inplace.++        See Also+        --------+        pandas.to_datetime : Convert argument to datetime.+        pandas.to_timedelta : Convert argument to timedelta.+        pandas.to_numeric : Convert argument to numeric type.++        Returns+        -------+        converted : same as input object+        """+        msg = ("convert_objects is deprecated.  To re-infer data dtypes for "+               "object columns, use {klass}.infer_objects()\nFor all "+               "other conversions use the data-type specific converters "+               "pd.to_datetime, pd.to_timedelta and pd.to_numeric."+               ).format(klass=self.__class__.__name__)+        warnings.warn(msg, FutureWarning, stacklevel=2)++        return self._constructor(+            self._data.convert(convert_dates=convert_dates,+                               convert_numeric=convert_numeric,+                               convert_timedeltas=convert_timedeltas,+                               copy=copy)).__finalize__(self)++    def infer_objects(self):+        """+        Attempt to infer better dtypes for object columns.++        Attempts soft conversion of object-dtyped+        columns, leaving non-object and unconvertible+        columns unchanged. The inference rules are the+        same as during normal Series/DataFrame construction.++        .. versionadded:: 0.21.0++        See Also+        --------+        pandas.to_datetime : Convert argument to datetime.+        pandas.to_timedelta : Convert argument to timedelta.+        pandas.to_numeric : Convert argument to numeric type.++        Returns+        -------+        converted : same type as input object++        Examples+        --------+        >>> df = pd.DataFrame({"A": ["a", 1, 2, 3]})+        >>> df = df.iloc[1:]+        >>> df+           A+        1  1+        2  2+        3  3++        >>> df.dtypes+        A    object+        dtype: object++        >>> df.infer_objects().dtypes+        A    int64+        dtype: object+        """+        # numeric=False necessary to only soft convert;+        # python objects will still be converted to+        # native numpy numeric types+        return self._constructor(+            self._data.convert(datetime=True, numeric=False,+                               timedelta=True, coerce=False,+                               copy=True)).__finalize__(self)++    # ----------------------------------------------------------------------+    # Filling NA's++    def fillna(self, value=None, method=None, axis=None, inplace=False,+               limit=None, downcast=None):+        """+        Fill NA/NaN values using the specified method++        Parameters+        ----------+        value : scalar, dict, Series, or DataFrame+            Value to use to fill holes (e.g. 0), alternately a+            dict/Series/DataFrame of values specifying which value to use for+            each index (for a Series) or column (for a DataFrame). (values not+            in the dict/Series/DataFrame will not be filled). This value cannot+            be a list.+        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None+            Method to use for filling holes in reindexed Series+            pad / ffill: propagate last valid observation forward to next valid+            backfill / bfill: use NEXT valid observation to fill gap+        axis : %(axes_single_arg)s+        inplace : boolean, default False+            If True, fill in place. Note: this will modify any+            other views on this object, (e.g. a no-copy slice for a column in a+            DataFrame).+        limit : int, default None+            If method is specified, this is the maximum number of consecutive+            NaN values to forward/backward fill. In other words, if there is+            a gap with more than this number of consecutive NaNs, it will only+            be partially filled. If method is not specified, this is the+            maximum number of entries along the entire axis where NaNs will be+            filled. Must be greater than 0 if not None.+        downcast : dict, default is None+            a dict of item->dtype of what to downcast if possible,+            or the string 'infer' which will try to downcast to an appropriate+            equal type (e.g. float64 to int64 if possible)++        See Also+        --------+        interpolate : Fill NaN values using interpolation.+        reindex, asfreq++        Returns+        -------+        filled : %(klass)s++        Examples+        --------+        >>> df = pd.DataFrame([[np.nan, 2, np.nan, 0],+        ...                    [3, 4, np.nan, 1],+        ...                    [np.nan, np.nan, np.nan, 5],+        ...                    [np.nan, 3, np.nan, 4]],+        ...                    columns=list('ABCD'))+        >>> df+             A    B   C  D+        0  NaN  2.0 NaN  0+        1  3.0  4.0 NaN  1+        2  NaN  NaN NaN  5+        3  NaN  3.0 NaN  4++        Replace all NaN elements with 0s.++        >>> df.fillna(0)+            A   B   C   D+        0   0.0 2.0 0.0 0+        1   3.0 4.0 0.0 1+        2   0.0 0.0 0.0 5+        3   0.0 3.0 0.0 4++        We can also propagate non-null values forward or backward.++        >>> df.fillna(method='ffill')+            A   B   C   D+        0   NaN 2.0 NaN 0+        1   3.0 4.0 NaN 1+        2   3.0 4.0 NaN 5+        3   3.0 3.0 NaN 4++        Replace all NaN elements in column 'A', 'B', 'C', and 'D', with 0, 1,+        2, and 3 respectively.++        >>> values = {'A': 0, 'B': 1, 'C': 2, 'D': 3}+        >>> df.fillna(value=values)+            A   B   C   D+        0   0.0 2.0 2.0 0+        1   3.0 4.0 2.0 1+        2   0.0 1.0 2.0 5+        3   0.0 3.0 2.0 4++        Only replace the first NaN element.++        >>> df.fillna(value=values, limit=1)+            A   B   C   D+        0   0.0 2.0 2.0 0+        1   3.0 4.0 NaN 1+        2   NaN 1.0 NaN 5+        3   NaN 3.0 NaN 4+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        value, method = validate_fillna_kwargs(value, method)++        self._consolidate_inplace()++        # set the default here, so functions examining the signaure+        # can detect if something was set (e.g. in groupby) (GH9221)+        if axis is None:+            axis = 0+        axis = self._get_axis_number(axis)++        from pandas import DataFrame+        if value is None:++            if self._is_mixed_type and axis == 1:+                if inplace:+                    raise NotImplementedError()+                result = self.T.fillna(method=method, limit=limit).T++                # need to downcast here because of all of the transposes+                result._data = result._data.downcast()++                return result++            # > 3d+            if self.ndim > 3:+                raise NotImplementedError('Cannot fillna with a method for > '+                                          '3dims')++            # 3d+            elif self.ndim == 3:+                # fill in 2d chunks+                result = {col: s.fillna(method=method, value=value)+                          for col, s in self.iteritems()}+                new_obj = self._constructor.\+                    from_dict(result).__finalize__(self)+                new_data = new_obj._data++            else:+                # 2d or less+                new_data = self._data.interpolate(method=method, axis=axis,+                                                  limit=limit, inplace=inplace,+                                                  coerce=True,+                                                  downcast=downcast)+        else:+            if len(self._get_axis(axis)) == 0:+                return self++            if self.ndim == 1:+                if isinstance(value, (dict, ABCSeries)):+                    from pandas import Series+                    value = Series(value)+                elif not is_list_like(value):+                    pass+                else:+                    raise TypeError('"value" parameter must be a scalar, dict '+                                    'or Series, but you passed a '+                                    '"{0}"'.format(type(value).__name__))++                new_data = self._data.fillna(value=value, limit=limit,+                                             inplace=inplace,+                                             downcast=downcast)++            elif isinstance(value, (dict, ABCSeries)):+                if axis == 1:+                    raise NotImplementedError('Currently only can fill '+                                              'with dict/Series column '+                                              'by column')++                result = self if inplace else self.copy()+                for k, v in compat.iteritems(value):+                    if k not in result:+                        continue+                    obj = result[k]+                    obj.fillna(v, limit=limit, inplace=True, downcast=downcast)+                return result if not inplace else None++            elif not is_list_like(value):+                new_data = self._data.fillna(value=value, limit=limit,+                                             inplace=inplace,+                                             downcast=downcast)+            elif isinstance(value, DataFrame) and self.ndim == 2:+                new_data = self.where(self.notna(), value)+            else:+                raise ValueError("invalid fill value with a %s" % type(value))++        if inplace:+            self._update_inplace(new_data)+        else:+            return self._constructor(new_data).__finalize__(self)++    def ffill(self, axis=None, inplace=False, limit=None, downcast=None):+        """+        Synonym for :meth:`DataFrame.fillna(method='ffill') <DataFrame.fillna>`+        """+        return self.fillna(method='ffill', axis=axis, inplace=inplace,+                           limit=limit, downcast=downcast)++    def bfill(self, axis=None, inplace=False, limit=None, downcast=None):+        """+        Synonym for :meth:`DataFrame.fillna(method='bfill') <DataFrame.fillna>`+        """+        return self.fillna(method='bfill', axis=axis, inplace=inplace,+                           limit=limit, downcast=downcast)++    _shared_docs['replace'] = ("""+        Replace values given in `to_replace` with `value`.++        Values of the %(klass)s are replaced with other values dynamically.+        This differs from updating with ``.loc`` or ``.iloc``, which require+        you to specify a location to update with some value.++        Parameters+        ----------+        to_replace : str, regex, list, dict, Series, int, float, or None+            How to find the values that will be replaced.++            * numeric, str or regex:++                - numeric: numeric values equal to `to_replace` will be+                  replaced with `value`+                - str: string exactly matching `to_replace` will be replaced+                  with `value`+                - regex: regexs matching `to_replace` will be replaced with+                  `value`++            * list of str, regex, or numeric:++                - First, if `to_replace` and `value` are both lists, they+                  **must** be the same length.+                - Second, if ``regex=True`` then all of the strings in **both**+                  lists will be interpreted as regexs otherwise they will match+                  directly. This doesn't matter much for `value` since there+                  are only a few possible substitution regexes you can use.+                - str, regex and numeric rules apply as above.++            * dict:++                - Dicts can be used to specify different replacement values+                  for different existing values. For example,+                  ``{'a': 'b', 'y': 'z'}`` replaces the value 'a' with 'b' and+                  'y' with 'z'. To use a dict in this way the `value`+                  parameter should be `None`.+                - For a DataFrame a dict can specify that different values+                  should be replaced in different columns. For example,+                  ``{'a': 1, 'b': 'z'}`` looks for the value 1 in column 'a'+                  and the value 'z' in column 'b' and replaces these values+                  with whatever is specified in `value`. The `value` parameter+                  should not be ``None`` in this case. You can treat this as a+                  special case of passing two lists except that you are+                  specifying the column to search in.+                - For a DataFrame nested dictionaries, e.g.,+                  ``{'a': {'b': np.nan}}``, are read as follows: look in column+                  'a' for the value 'b' and replace it with NaN. The `value`+                  parameter should be ``None`` to use a nested dict in this+                  way. You can nest regular expressions as well. Note that+                  column names (the top-level dictionary keys in a nested+                  dictionary) **cannot** be regular expressions.++            * None:++                - This means that the `regex` argument must be a string,+                  compiled regular expression, or list, dict, ndarray or+                  Series of such elements. If `value` is also ``None`` then+                  this **must** be a nested dictionary or Series.++            See the examples section for examples of each of these.+        value : scalar, dict, list, str, regex, default None+            Value to replace any values matching `to_replace` with.+            For a DataFrame a dict of values can be used to specify which+            value to use for each column (columns not in the dict will not be+            filled). Regular expressions, strings and lists or dicts of such+            objects are also allowed.+        inplace : boolean, default False+            If True, in place. Note: this will modify any+            other views on this object (e.g. a column from a DataFrame).+            Returns the caller if this is True.+        limit : int, default None+            Maximum size gap to forward or backward fill.+        regex : bool or same types as `to_replace`, default False+            Whether to interpret `to_replace` and/or `value` as regular+            expressions. If this is ``True`` then `to_replace` *must* be a+            string. Alternatively, this could be a regular expression or a+            list, dict, or array of regular expressions in which case+            `to_replace` must be ``None``.+        method : {'pad', 'ffill', 'bfill', `None`}+            The method to use when for replacement, when `to_replace` is a+            scalar, list or tuple and `value` is ``None``.++            .. versionchanged:: 0.23.0+                Added to DataFrame.++        See Also+        --------+        %(klass)s.fillna : Fill NA values+        %(klass)s.where : Replace values based on boolean condition+        Series.str.replace : Simple string replacement.++        Returns+        -------+        %(klass)s+            Object after replacement.++        Raises+        ------+        AssertionError+            * If `regex` is not a ``bool`` and `to_replace` is not+              ``None``.+        TypeError+            * If `to_replace` is a ``dict`` and `value` is not a ``list``,+              ``dict``, ``ndarray``, or ``Series``+            * If `to_replace` is ``None`` and `regex` is not compilable+              into a regular expression or is a list, dict, ndarray, or+              Series.+            * When replacing multiple ``bool`` or ``datetime64`` objects and+              the arguments to `to_replace` does not match the type of the+              value being replaced+        ValueError+            * If a ``list`` or an ``ndarray`` is passed to `to_replace` and+              `value` but they are not the same length.++        Notes+        -----+        * Regex substitution is performed under the hood with ``re.sub``. The+          rules for substitution for ``re.sub`` are the same.+        * Regular expressions will only substitute on strings, meaning you+          cannot provide, for example, a regular expression matching floating+          point numbers and expect the columns in your frame that have a+          numeric dtype to be matched. However, if those floating point+          numbers *are* strings, then you can do this.+        * This method has *a lot* of options. You are encouraged to experiment+          and play with this method to gain intuition about how it works.+        * When dict is used as the `to_replace` value, it is like+          key(s) in the dict are the to_replace part and+          value(s) in the dict are the value parameter.++        Examples+        --------++        **Scalar `to_replace` and `value`**++        >>> s = pd.Series([0, 1, 2, 3, 4])+        >>> s.replace(0, 5)+        0    5+        1    1+        2    2+        3    3+        4    4+        dtype: int64++        >>> df = pd.DataFrame({'A': [0, 1, 2, 3, 4],+        ...                    'B': [5, 6, 7, 8, 9],+        ...                    'C': ['a', 'b', 'c', 'd', 'e']})+        >>> df.replace(0, 5)+           A  B  C+        0  5  5  a+        1  1  6  b+        2  2  7  c+        3  3  8  d+        4  4  9  e++        **List-like `to_replace`**++        >>> df.replace([0, 1, 2, 3], 4)+           A  B  C+        0  4  5  a+        1  4  6  b+        2  4  7  c+        3  4  8  d+        4  4  9  e++        >>> df.replace([0, 1, 2, 3], [4, 3, 2, 1])+           A  B  C+        0  4  5  a+        1  3  6  b+        2  2  7  c+        3  1  8  d+        4  4  9  e++        >>> s.replace([1, 2], method='bfill')+        0    0+        1    3+        2    3+        3    3+        4    4+        dtype: int64++        **dict-like `to_replace`**++        >>> df.replace({0: 10, 1: 100})+             A  B  C+        0   10  5  a+        1  100  6  b+        2    2  7  c+        3    3  8  d+        4    4  9  e++        >>> df.replace({'A': 0, 'B': 5}, 100)+             A    B  C+        0  100  100  a+        1    1    6  b+        2    2    7  c+        3    3    8  d+        4    4    9  e++        >>> df.replace({'A': {0: 100, 4: 400}})+             A  B  C+        0  100  5  a+        1    1  6  b+        2    2  7  c+        3    3  8  d+        4  400  9  e++        **Regular expression `to_replace`**++        >>> df = pd.DataFrame({'A': ['bat', 'foo', 'bait'],+        ...                    'B': ['abc', 'bar', 'xyz']})+        >>> df.replace(to_replace=r'^ba.$', value='new', regex=True)+              A    B+        0   new  abc+        1   foo  new+        2  bait  xyz++        >>> df.replace({'A': r'^ba.$'}, {'A': 'new'}, regex=True)+              A    B+        0   new  abc+        1   foo  bar+        2  bait  xyz++        >>> df.replace(regex=r'^ba.$', value='new')+              A    B+        0   new  abc+        1   foo  new+        2  bait  xyz++        >>> df.replace(regex={r'^ba.$':'new', 'foo':'xyz'})+              A    B+        0   new  abc+        1   xyz  new+        2  bait  xyz++        >>> df.replace(regex=[r'^ba.$', 'foo'], value='new')+              A    B+        0   new  abc+        1   new  new+        2  bait  xyz++        Note that when replacing multiple ``bool`` or ``datetime64`` objects,+        the data types in the `to_replace` parameter must match the data+        type of the value being replaced:++        >>> df = pd.DataFrame({'A': [True, False, True],+        ...                    'B': [False, True, False]})+        >>> df.replace({'a string': 'new value', True: False})  # raises+        Traceback (most recent call last):+            ...+        TypeError: Cannot compare types 'ndarray(dtype=bool)' and 'str'++        This raises a ``TypeError`` because one of the ``dict`` keys is not of+        the correct type for replacement.++        Compare the behavior of ``s.replace({'a': None})`` and+        ``s.replace('a', None)`` to understand the peculiarities+        of the `to_replace` parameter:++        >>> s = pd.Series([10, 'a', 'a', 'b', 'a'])++        When one uses a dict as the `to_replace` value, it is like the+        value(s) in the dict are equal to the `value` parameter.+        ``s.replace({'a': None})`` is equivalent to+        ``s.replace(to_replace={'a': None}, value=None, method=None)``:++        >>> s.replace({'a': None})+        0      10+        1    None+        2    None+        3       b+        4    None+        dtype: object++        When ``value=None`` and `to_replace` is a scalar, list or+        tuple, `replace` uses the method parameter (default 'pad') to do the+        replacement. So this is why the 'a' values are being replaced by 10+        in rows 1 and 2 and 'b' in row 4 in this case.+        The command ``s.replace('a', None)`` is actually equivalent to+        ``s.replace(to_replace='a', value=None, method='pad')``:++        >>> s.replace('a', None)+        0    10+        1    10+        2    10+        3     b+        4     b+        dtype: object+    """)++    @Appender(_shared_docs['replace'] % _shared_doc_kwargs)+    def replace(self, to_replace=None, value=None, inplace=False, limit=None,+                regex=False, method='pad'):+        inplace = validate_bool_kwarg(inplace, 'inplace')+        if not is_bool(regex) and to_replace is not None:+            raise AssertionError("'to_replace' must be 'None' if 'regex' is "+                                 "not a bool")++        self._consolidate_inplace()++        if value is None:+            # passing a single value that is scalar like+            # when value is None (GH5319), for compat+            if not is_dict_like(to_replace) and not is_dict_like(regex):+                to_replace = [to_replace]++            if isinstance(to_replace, (tuple, list)):+                if isinstance(self, pd.DataFrame):+                    return self.apply(_single_replace,+                                      args=(to_replace, method, inplace,+                                            limit))+                return _single_replace(self, to_replace, method, inplace,+                                       limit)++            if not is_dict_like(to_replace):+                if not is_dict_like(regex):+                    raise TypeError('If "to_replace" and "value" are both None'+                                    ' and "to_replace" is not a list, then '+                                    'regex must be a mapping')+                to_replace = regex+                regex = True++            items = list(compat.iteritems(to_replace))+            keys, values = lzip(*items) or ([], [])++            are_mappings = [is_dict_like(v) for v in values]++            if any(are_mappings):+                if not all(are_mappings):+                    raise TypeError("If a nested mapping is passed, all values"+                                    " of the top level mapping must be "+                                    "mappings")+                # passed a nested dict/Series+                to_rep_dict = {}+                value_dict = {}++                for k, v in items:+                    keys, values = lzip(*v.items()) or ([], [])+                    if set(keys) & set(values):+                        raise ValueError("Replacement not allowed with "+                                         "overlapping keys and values")+                    to_rep_dict[k] = list(keys)+                    value_dict[k] = list(values)++                to_replace, value = to_rep_dict, value_dict+            else:+                to_replace, value = keys, values++            return self.replace(to_replace, value, inplace=inplace,+                                limit=limit, regex=regex)+        else:++            # need a non-zero len on all axes+            for a in self._AXIS_ORDERS:+                if not len(self._get_axis(a)):+                    return self++            new_data = self._data+            if is_dict_like(to_replace):+                if is_dict_like(value):  # {'A' : NA} -> {'A' : 0}+                    res = self if inplace else self.copy()+                    for c, src in compat.iteritems(to_replace):+                        if c in value and c in self:+                            # object conversion is handled in+                            # series.replace which is called recursivelly+                            res[c] = res[c].replace(to_replace=src,+                                                    value=value[c],+                                                    inplace=False,+                                                    regex=regex)+                    return None if inplace else res++                # {'A': NA} -> 0+                elif not is_list_like(value):+                    keys = [(k, src) for k, src in compat.iteritems(to_replace)+                            if k in self]+                    keys_len = len(keys) - 1+                    for i, (k, src) in enumerate(keys):+                        convert = i == keys_len+                        new_data = new_data.replace(to_replace=src,+                                                    value=value,+                                                    filter=[k],+                                                    inplace=inplace,+                                                    regex=regex,+                                                    convert=convert)+                else:+                    raise TypeError('value argument must be scalar, dict, or '+                                    'Series')++            elif is_list_like(to_replace):  # [NA, ''] -> [0, 'missing']+                if is_list_like(value):+                    if len(to_replace) != len(value):+                        raise ValueError('Replacement lists must match '+                                         'in length. Expecting %d got %d ' %+                                         (len(to_replace), len(value)))++                    new_data = self._data.replace_list(src_list=to_replace,+                                                       dest_list=value,+                                                       inplace=inplace,+                                                       regex=regex)++                else:  # [NA, ''] -> 0+                    new_data = self._data.replace(to_replace=to_replace,+                                                  value=value, inplace=inplace,+                                                  regex=regex)+            elif to_replace is None:+                if not (is_re_compilable(regex) or+                        is_list_like(regex) or is_dict_like(regex)):+                    raise TypeError("'regex' must be a string or a compiled "+                                    "regular expression or a list or dict of "+                                    "strings or regular expressions, you "+                                    "passed a"+                                    " {0!r}".format(type(regex).__name__))+                return self.replace(regex, value, inplace=inplace, limit=limit,+                                    regex=True)+            else:++                # dest iterable dict-like+                if is_dict_like(value):  # NA -> {'A' : 0, 'B' : -1}+                    new_data = self._data++                    for k, v in compat.iteritems(value):+                        if k in self:+                            new_data = new_data.replace(to_replace=to_replace,+                                                        value=v, filter=[k],+                                                        inplace=inplace,+                                                        regex=regex)++                elif not is_list_like(value):  # NA -> 0+                    new_data = self._data.replace(to_replace=to_replace,+                                                  value=value, inplace=inplace,+                                                  regex=regex)+                else:+                    msg = ('Invalid "to_replace" type: '+                           '{0!r}').format(type(to_replace).__name__)+                    raise TypeError(msg)  # pragma: no cover++        if inplace:+            self._update_inplace(new_data)+        else:+            return self._constructor(new_data).__finalize__(self)++    _shared_docs['interpolate'] = """+        Please note that only ``method='linear'`` is supported for+        DataFrame/Series with a MultiIndex.++        Parameters+        ----------+        method : str, default 'linear'+            Interpolation technique to use. One of:++            * 'linear': Ignore the index and treat the values as equally+              spaced. This is the only method supported on MultiIndexes.+            * 'time': Works on daily and higher resolution data to interpolate+              given length of interval.+            * 'index', 'values': use the actual numerical values of the index.+            * 'pad': Fill in NaNs using existing values.+            * 'nearest', 'zero', 'slinear', 'quadratic', 'cubic', 'spline',+              'barycentric', 'polynomial': Passed to+              `scipy.interpolate.interp1d`. Both 'polynomial' and 'spline'+              require that you also specify an `order` (int),+              e.g. ``df.interpolate(method='polynomial', order=4)``.+              These use the numerical values of the index.+            * 'krogh', 'piecewise_polynomial', 'spline', 'pchip', 'akima':+              Wrappers around the SciPy interpolation methods of similar+              names. See `Notes`.+            * 'from_derivatives': Refers to+              `scipy.interpolate.BPoly.from_derivatives` which+              replaces 'piecewise_polynomial' interpolation method in+              scipy 0.18.++            .. versionadded:: 0.18.1++               Added support for the 'akima' method.+               Added interpolate method 'from_derivatives' which replaces+               'piecewise_polynomial' in SciPy 0.18; backwards-compatible with+               SciPy < 0.18++        axis : {0 or 'index', 1 or 'columns', None}, default None+            Axis to interpolate along.+        limit : int, optional+            Maximum number of consecutive NaNs to fill. Must be greater than+            0.+        inplace : bool, default False+            Update the data in place if possible.+        limit_direction : {'forward', 'backward', 'both'}, default 'forward'+            If limit is specified, consecutive NaNs will be filled in this+            direction.+        limit_area : {`None`, 'inside', 'outside'}, default None+            If limit is specified, consecutive NaNs will be filled with this+            restriction.++            * ``None``: No fill restriction.+            * 'inside': Only fill NaNs surrounded by valid values+              (interpolate).+            * 'outside': Only fill NaNs outside valid values (extrapolate).++            .. versionadded:: 0.21.0++        downcast : optional, 'infer' or None, defaults to None+            Downcast dtypes if possible.+        **kwargs+            Keyword arguments to pass on to the interpolating function.++        Returns+        -------+        Series or DataFrame+            Returns the same object type as the caller, interpolated at+            some or all ``NaN`` values++        See Also+        --------+        fillna : Fill missing values using different methods.+        scipy.interpolate.Akima1DInterpolator : Piecewise cubic polynomials+            (Akima interpolator).+        scipy.interpolate.BPoly.from_derivatives : Piecewise polynomial in the+            Bernstein basis.+        scipy.interpolate.interp1d : Interpolate a 1-D function.+        scipy.interpolate.KroghInterpolator : Interpolate polynomial (Krogh+            interpolator).+        scipy.interpolate.PchipInterpolator : PCHIP 1-d monotonic cubic+            interpolation.+        scipy.interpolate.CubicSpline : Cubic spline data interpolator.++        Notes+        -----+        The 'krogh', 'piecewise_polynomial', 'spline', 'pchip' and 'akima'+        methods are wrappers around the respective SciPy implementations of+        similar names. These use the actual numerical values of the index.+        For more information on their behavior, see the+        `SciPy documentation+        <http://docs.scipy.org/doc/scipy/reference/interpolate.html#univariate-interpolation>`__+        and `SciPy tutorial+        <http://docs.scipy.org/doc/scipy/reference/tutorial/interpolate.html>`__.++        Examples+        --------+        Filling in ``NaN`` in a :class:`~pandas.Series` via linear+        interpolation.++        >>> s = pd.Series([0, 1, np.nan, 3])+        >>> s+        0    0.0+        1    1.0+        2    NaN+        3    3.0+        dtype: float64+        >>> s.interpolate()+        0    0.0+        1    1.0+        2    2.0+        3    3.0+        dtype: float64++        Filling in ``NaN`` in a Series by padding, but filling at most two+        consecutive ``NaN`` at a time.++        >>> s = pd.Series([np.nan, "single_one", np.nan,+        ...                "fill_two_more", np.nan, np.nan, np.nan,+        ...                4.71, np.nan])+        >>> s+        0              NaN+        1       single_one+        2              NaN+        3    fill_two_more+        4              NaN+        5              NaN+        6              NaN+        7             4.71+        8              NaN+        dtype: object+        >>> s.interpolate(method='pad', limit=2)+        0              NaN+        1       single_one+        2       single_one+        3    fill_two_more+        4    fill_two_more+        5    fill_two_more+        6              NaN+        7             4.71+        8             4.71+        dtype: object++        Filling in ``NaN`` in a Series via polynomial interpolation or splines:+        Both 'polynomial' and 'spline' methods require that you also specify+        an ``order`` (int).++        >>> s = pd.Series([0, 2, np.nan, 8])+        >>> s.interpolate(method='polynomial', order=2)+        0    0.000000+        1    2.000000+        2    4.666667+        3    8.000000+        dtype: float64++        Fill the DataFrame forward (that is, going down) along each column+        using linear interpolation.++        Note how the last entry in column 'a' is interpolated differently,+        because there is no entry after it to use for interpolation.+        Note how the first entry in column 'b' remains ``NaN``, because there+        is no entry befofe it to use for interpolation.++        >>> df = pd.DataFrame([(0.0,  np.nan, -1.0, 1.0),+        ...                    (np.nan, 2.0, np.nan, np.nan),+        ...                    (2.0, 3.0, np.nan, 9.0),+        ...                    (np.nan, 4.0, -4.0, 16.0)],+        ...                   columns=list('abcd'))+        >>> df+             a    b    c     d+        0  0.0  NaN -1.0   1.0+        1  NaN  2.0  NaN   NaN+        2  2.0  3.0  NaN   9.0+        3  NaN  4.0 -4.0  16.0+        >>> df.interpolate(method='linear', limit_direction='forward', axis=0)+             a    b    c     d+        0  0.0  NaN -1.0   1.0+        1  1.0  2.0 -2.0   5.0+        2  2.0  3.0 -3.0   9.0+        3  2.0  4.0 -4.0  16.0++        Using polynomial interpolation.++        >>> df['d'].interpolate(method='polynomial', order=2)+        0     1.0+        1     4.0+        2     9.0+        3    16.0+        Name: d, dtype: float64+        """++    @Appender(_shared_docs['interpolate'] % _shared_doc_kwargs)+    def interpolate(self, method='linear', axis=0, limit=None, inplace=False,+                    limit_direction='forward', limit_area=None,+                    downcast=None, **kwargs):+        """+        Interpolate values according to different methods.+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')++        if self.ndim > 2:+            raise NotImplementedError("Interpolate has not been implemented "+                                      "on Panel and Panel 4D objects.")++        if axis == 0:+            ax = self._info_axis_name+            _maybe_transposed_self = self+        elif axis == 1:+            _maybe_transposed_self = self.T+            ax = 1+        else:+            _maybe_transposed_self = self+        ax = _maybe_transposed_self._get_axis_number(ax)++        if _maybe_transposed_self.ndim == 2:+            alt_ax = 1 - ax+        else:+            alt_ax = ax++        if (isinstance(_maybe_transposed_self.index, MultiIndex) and+                method != 'linear'):+            raise ValueError("Only `method=linear` interpolation is supported "+                             "on MultiIndexes.")++        if _maybe_transposed_self._data.get_dtype_counts().get(+                'object') == len(_maybe_transposed_self.T):+            raise TypeError("Cannot interpolate with all NaNs.")++        # create/use the index+        if method == 'linear':+            # prior default+            index = np.arange(len(_maybe_transposed_self._get_axis(alt_ax)))+        else:+            index = _maybe_transposed_self._get_axis(alt_ax)++        if isna(index).any():+            raise NotImplementedError("Interpolation with NaNs in the index "+                                      "has not been implemented. Try filling "+                                      "those NaNs before interpolating.")+        data = _maybe_transposed_self._data+        new_data = data.interpolate(method=method, axis=ax, index=index,+                                    values=_maybe_transposed_self, limit=limit,+                                    limit_direction=limit_direction,+                                    limit_area=limit_area,+                                    inplace=inplace, downcast=downcast,+                                    **kwargs)++        if inplace:+            if axis == 1:+                new_data = self._constructor(new_data).T._data+            self._update_inplace(new_data)+        else:+            res = self._constructor(new_data).__finalize__(self)+            if axis == 1:+                res = res.T+            return res++    # ----------------------------------------------------------------------+    # Timeseries methods Methods++    def asof(self, where, subset=None):+        """+        The last row without any NaN is taken (or the last row without+        NaN considering only the subset of columns in the case of a DataFrame)++        .. versionadded:: 0.19.0 For DataFrame++        If there is no good value, NaN is returned for a Series+        a Series of NaN values for a DataFrame++        Parameters+        ----------+        where : date or array of dates+        subset : string or list of strings, default None+           if not None use these columns for NaN propagation++        Notes+        -----+        Dates are assumed to be sorted+        Raises if this is not the case++        Returns+        -------+        where is scalar++          - value or NaN if input is Series+          - Series if input is DataFrame++        where is Index: same shape object as input++        See Also+        --------+        merge_asof++        """++        if isinstance(where, compat.string_types):+            from pandas import to_datetime+            where = to_datetime(where)++        if not self.index.is_monotonic:+            raise ValueError("asof requires a sorted index")++        is_series = isinstance(self, ABCSeries)+        if is_series:+            if subset is not None:+                raise ValueError("subset is not valid for Series")+        elif self.ndim > 2:+            raise NotImplementedError("asof is not implemented "+                                      "for {type}".format(type=type(self)))+        else:+            if subset is None:+                subset = self.columns+            if not is_list_like(subset):+                subset = [subset]++        is_list = is_list_like(where)+        if not is_list:+            start = self.index[0]+            if isinstance(self.index, PeriodIndex):+                where = Period(where, freq=self.index.freq).ordinal+                start = start.ordinal++            if where < start:+                if not is_series:+                    from pandas import Series+                    return Series(index=self.columns, name=where)+                return np.nan++            # It's always much faster to use a *while* loop here for+            # Series than pre-computing all the NAs. However a+            # *while* loop is extremely expensive for DataFrame+            # so we later pre-compute all the NAs and use the same+            # code path whether *where* is a scalar or list.+            # See PR: https://github.com/pandas-dev/pandas/pull/14476+            if is_series:+                loc = self.index.searchsorted(where, side='right')+                if loc > 0:+                    loc -= 1++                values = self._values+                while loc > 0 and isna(values[loc]):+                    loc -= 1+                return values[loc]++        if not isinstance(where, Index):+            where = Index(where) if is_list else Index([where])++        nulls = self.isna() if is_series else self[subset].isna().any(1)+        if nulls.all():+            if is_series:+                return self._constructor(np.nan, index=where, name=self.name)+            elif is_list:+                from pandas import DataFrame+                return DataFrame(np.nan, index=where, columns=self.columns)+            else:+                from pandas import Series+                return Series(np.nan, index=self.columns, name=where[0])++        locs = self.index.asof_locs(where, ~(nulls.values))++        # mask the missing+        missing = locs == -1+        data = self.take(locs, is_copy=False)+        data.index = where+        data.loc[missing] = np.nan+        return data if is_list else data.iloc[-1]++    # ----------------------------------------------------------------------+    # Action Methods++    _shared_docs['isna'] = """+        Detect missing values.++        Return a boolean same-sized object indicating if the values are NA.+        NA values, such as None or :attr:`numpy.NaN`, gets mapped to True+        values.+        Everything else gets mapped to False values. Characters such as empty+        strings ``''`` or :attr:`numpy.inf` are not considered NA values+        (unless you set ``pandas.options.mode.use_inf_as_na = True``).++        Returns+        -------+        %(klass)s+            Mask of bool values for each element in %(klass)s that+            indicates whether an element is not an NA value.++        See Also+        --------+        %(klass)s.isnull : alias of isna+        %(klass)s.notna : boolean inverse of isna+        %(klass)s.dropna : omit axes labels with missing values+        isna : top-level isna++        Examples+        --------+        Show which entries in a DataFrame are NA.++        >>> df = pd.DataFrame({'age': [5, 6, np.NaN],+        ...                    'born': [pd.NaT, pd.Timestamp('1939-05-27'),+        ...                             pd.Timestamp('1940-04-25')],+        ...                    'name': ['Alfred', 'Batman', ''],+        ...                    'toy': [None, 'Batmobile', 'Joker']})+        >>> df+           age       born    name        toy+        0  5.0        NaT  Alfred       None+        1  6.0 1939-05-27  Batman  Batmobile+        2  NaN 1940-04-25              Joker++        >>> df.isna()+             age   born   name    toy+        0  False   True  False   True+        1  False  False  False  False+        2   True  False  False  False++        Show which entries in a Series are NA.++        >>> ser = pd.Series([5, 6, np.NaN])+        >>> ser+        0    5.0+        1    6.0+        2    NaN+        dtype: float64++        >>> ser.isna()+        0    False+        1    False+        2     True+        dtype: bool+        """++    @Appender(_shared_docs['isna'] % _shared_doc_kwargs)+    def isna(self):+        return isna(self).__finalize__(self)++    @Appender(_shared_docs['isna'] % _shared_doc_kwargs)+    def isnull(self):+        return isna(self).__finalize__(self)++    _shared_docs['notna'] = """+        Detect existing (non-missing) values.++        Return a boolean same-sized object indicating if the values are not NA.+        Non-missing values get mapped to True. Characters such as empty+        strings ``''`` or :attr:`numpy.inf` are not considered NA values+        (unless you set ``pandas.options.mode.use_inf_as_na = True``).+        NA values, such as None or :attr:`numpy.NaN`, get mapped to False+        values.++        Returns+        -------+        %(klass)s+            Mask of bool values for each element in %(klass)s that+            indicates whether an element is not an NA value.++        See Also+        --------+        %(klass)s.notnull : alias of notna+        %(klass)s.isna : boolean inverse of notna+        %(klass)s.dropna : omit axes labels with missing values+        notna : top-level notna++        Examples+        --------+        Show which entries in a DataFrame are not NA.++        >>> df = pd.DataFrame({'age': [5, 6, np.NaN],+        ...                    'born': [pd.NaT, pd.Timestamp('1939-05-27'),+        ...                             pd.Timestamp('1940-04-25')],+        ...                    'name': ['Alfred', 'Batman', ''],+        ...                    'toy': [None, 'Batmobile', 'Joker']})+        >>> df+           age       born    name        toy+        0  5.0        NaT  Alfred       None+        1  6.0 1939-05-27  Batman  Batmobile+        2  NaN 1940-04-25              Joker++        >>> df.notna()+             age   born  name    toy+        0   True  False  True  False+        1   True   True  True   True+        2  False   True  True   True++        Show which entries in a Series are not NA.++        >>> ser = pd.Series([5, 6, np.NaN])+        >>> ser+        0    5.0+        1    6.0+        2    NaN+        dtype: float64++        >>> ser.notna()+        0     True+        1     True+        2    False+        dtype: bool+        """++    @Appender(_shared_docs['notna'] % _shared_doc_kwargs)+    def notna(self):+        return notna(self).__finalize__(self)++    @Appender(_shared_docs['notna'] % _shared_doc_kwargs)+    def notnull(self):+        return notna(self).__finalize__(self)++    def _clip_with_scalar(self, lower, upper, inplace=False):+        if ((lower is not None and np.any(isna(lower))) or+                (upper is not None and np.any(isna(upper)))):+            raise ValueError("Cannot use an NA value as a clip threshold")++        result = self.values+        mask = isna(result)++        with np.errstate(all='ignore'):+            if upper is not None:+                result = np.where(result >= upper, upper, result)+            if lower is not None:+                result = np.where(result <= lower, lower, result)+        if np.any(mask):+            result[mask] = np.nan++        axes_dict = self._construct_axes_dict()+        result = self._constructor(result, **axes_dict).__finalize__(self)++        if inplace:+            self._update_inplace(result)+        else:+            return result++    def _clip_with_one_bound(self, threshold, method, axis, inplace):++        inplace = validate_bool_kwarg(inplace, 'inplace')+        if axis is not None:+            axis = self._get_axis_number(axis)++        # method is self.le for upper bound and self.ge for lower bound+        if is_scalar(threshold) and is_number(threshold):+            if method.__name__ == 'le':+                return self._clip_with_scalar(None, threshold, inplace=inplace)+            return self._clip_with_scalar(threshold, None, inplace=inplace)++        subset = method(threshold, axis=axis) | isna(self)++        # GH #15390+        # In order for where method to work, the threshold must+        # be transformed to NDFrame from other array like structure.+        if (not isinstance(threshold, ABCSeries)) and is_list_like(threshold):+            if isinstance(self, ABCSeries):+                threshold = pd.Series(threshold, index=self.index)+            else:+                threshold = _align_method_FRAME(self, np.asarray(threshold),+                                                axis)+        return self.where(subset, threshold, axis=axis, inplace=inplace)++    def clip(self, lower=None, upper=None, axis=None, inplace=False,+             *args, **kwargs):+        """+        Trim values at input threshold(s).++        Assigns values outside boundary to boundary values. Thresholds+        can be singular values or array like, and in the latter case+        the clipping is performed element-wise in the specified axis.++        Parameters+        ----------+        lower : float or array_like, default None+            Minimum threshold value. All values below this+            threshold will be set to it.+        upper : float or array_like, default None+            Maximum threshold value. All values above this+            threshold will be set to it.+        axis : int or string axis name, optional+            Align object with lower and upper along the given axis.+        inplace : boolean, default False+            Whether to perform the operation in place on the data.++            .. versionadded:: 0.21.0+        *args, **kwargs+            Additional keywords have no effect but might be accepted+            for compatibility with numpy.++        See Also+        --------+        clip_lower : Clip values below specified threshold(s).+        clip_upper : Clip values above specified threshold(s).++        Returns+        -------+        Series or DataFrame+            Same type as calling object with the values outside the+            clip boundaries replaced++        Examples+        --------+        >>> data = {'col_0': [9, -3, 0, -1, 5], 'col_1': [-2, -7, 6, 8, -5]}+        >>> df = pd.DataFrame(data)+        >>> df+           col_0  col_1+        0      9     -2+        1     -3     -7+        2      0      6+        3     -1      8+        4      5     -5++        Clips per column using lower and upper thresholds:++        >>> df.clip(-4, 6)+           col_0  col_1+        0      6     -2+        1     -3     -4+        2      0      6+        3     -1      6+        4      5     -4++        Clips using specific lower and upper thresholds per column element:++        >>> t = pd.Series([2, -4, -1, 6, 3])+        >>> t+        0    2+        1   -4+        2   -1+        3    6+        4    3+        dtype: int64++        >>> df.clip(t, t + 4, axis=0)+           col_0  col_1+        0      6      2+        1     -3     -4+        2      0      3+        3      6      8+        4      5      3+        """+        if isinstance(self, ABCPanel):+            raise NotImplementedError("clip is not supported yet for panels")++        inplace = validate_bool_kwarg(inplace, 'inplace')++        axis = nv.validate_clip_with_axis(axis, args, kwargs)+        if axis is not None:+            axis = self._get_axis_number(axis)++        # GH 17276+        # numpy doesn't like NaN as a clip value+        # so ignore+        # GH 19992+        # numpy doesn't drop a list-like bound containing NaN+        if not is_list_like(lower) and np.any(pd.isnull(lower)):+            lower = None+        if not is_list_like(upper) and np.any(pd.isnull(upper)):+            upper = None++        # GH 2747 (arguments were reversed)+        if lower is not None and upper is not None:+            if is_scalar(lower) and is_scalar(upper):+                lower, upper = min(lower, upper), max(lower, upper)++        # fast-path for scalars+        if ((lower is None or (is_scalar(lower) and is_number(lower))) and+                (upper is None or (is_scalar(upper) and is_number(upper)))):+            return self._clip_with_scalar(lower, upper, inplace=inplace)++        result = self+        if lower is not None:+            result = result.clip_lower(lower, axis, inplace=inplace)+        if upper is not None:+            if inplace:+                result = self+            result = result.clip_upper(upper, axis, inplace=inplace)++        return result++    def clip_upper(self, threshold, axis=None, inplace=False):+        """+        Trim values above a given threshold.++        Elements above the `threshold` will be changed to match the+        `threshold` value(s). Threshold can be a single value or an array,+        in the latter case it performs the truncation element-wise.++        Parameters+        ----------+        threshold : numeric or array-like+            Maximum value allowed. All values above threshold will be set to+            this value.++            * float : every value is compared to `threshold`.+            * array-like : The shape of `threshold` should match the object+              it's compared to. When `self` is a Series, `threshold` should be+              the length. When `self` is a DataFrame, `threshold` should 2-D+              and the same shape as `self` for ``axis=None``, or 1-D and the+              same length as the axis being compared.++        axis : {0 or 'index', 1 or 'columns'}, default 0+            Align object with `threshold` along the given axis.+        inplace : boolean, default False+            Whether to perform the operation in place on the data.++            .. versionadded:: 0.21.0++        Returns+        -------+        clipped+            Original data with values trimmed.++        See Also+        --------+        DataFrame.clip : General purpose method to trim DataFrame values to+            given threshold(s)+        DataFrame.clip_lower : Trim DataFrame values below given+            threshold(s)+        Series.clip : General purpose method to trim Series values to given+            threshold(s)+        Series.clip_lower : Trim Series values below given threshold(s)++        Examples+        --------+        >>> s = pd.Series([1, 2, 3, 4, 5])+        >>> s+        0    1+        1    2+        2    3+        3    4+        4    5+        dtype: int64++        >>> s.clip_upper(3)+        0    1+        1    2+        2    3+        3    3+        4    3+        dtype: int64++        >>> t = [5, 4, 3, 2, 1]+        >>> t+        [5, 4, 3, 2, 1]++        >>> s.clip_upper(t)+        0    1+        1    2+        2    3+        3    2+        4    1+        dtype: int64+        """+        return self._clip_with_one_bound(threshold, method=self.le,+                                         axis=axis, inplace=inplace)++    def clip_lower(self, threshold, axis=None, inplace=False):+        """+        Trim values below a given threshold.++        Elements below the `threshold` will be changed to match the+        `threshold` value(s). Threshold can be a single value or an array,+        in the latter case it performs the truncation element-wise.++        Parameters+        ----------+        threshold : numeric or array-like+            Minimum value allowed. All values below threshold will be set to+            this value.++            * float : every value is compared to `threshold`.+            * array-like : The shape of `threshold` should match the object+              it's compared to. When `self` is a Series, `threshold` should be+              the length. When `self` is a DataFrame, `threshold` should 2-D+              and the same shape as `self` for ``axis=None``, or 1-D and the+              same length as the axis being compared.++        axis : {0 or 'index', 1 or 'columns'}, default 0+            Align `self` with `threshold` along the given axis.++        inplace : boolean, default False+            Whether to perform the operation in place on the data.++            .. versionadded:: 0.21.0++        Returns+        -------+        clipped+            Original data with values trimmed.++        See Also+        --------+        DataFrame.clip : General purpose method to trim DataFrame values to+            given threshold(s)+        DataFrame.clip_upper : Trim DataFrame values above given+            threshold(s)+        Series.clip : General purpose method to trim Series values to given+            threshold(s)+        Series.clip_upper : Trim Series values above given threshold(s)++        Examples+        --------++        Series single threshold clipping:++        >>> s = pd.Series([5, 6, 7, 8, 9])+        >>> s.clip_lower(8)+        0    8+        1    8+        2    8+        3    8+        4    9+        dtype: int64++        Series clipping element-wise using an array of thresholds. `threshold`+        should be the same length as the Series.++        >>> elemwise_thresholds = [4, 8, 7, 2, 5]+        >>> s.clip_lower(elemwise_thresholds)+        0    5+        1    8+        2    7+        3    8+        4    9+        dtype: int64++        DataFrames can be compared to a scalar.++        >>> df = pd.DataFrame({"A": [1, 3, 5], "B": [2, 4, 6]})+        >>> df+           A  B+        0  1  2+        1  3  4+        2  5  6++        >>> df.clip_lower(3)+           A  B+        0  3  3+        1  3  4+        2  5  6++        Or to an array of values. By default, `threshold` should be the same+        shape as the DataFrame.++        >>> df.clip_lower(np.array([[3, 4], [2, 2], [6, 2]]))+           A  B+        0  3  4+        1  3  4+        2  6  6++        Control how `threshold` is broadcast with `axis`. In this case+        `threshold` should be the same length as the axis specified by+        `axis`.++        >>> df.clip_lower([3, 3, 5], axis='index')+           A  B+        0  3  3+        1  3  4+        2  5  6++        >>> df.clip_lower([4, 5], axis='columns')+           A  B+        0  4  5+        1  4  5+        2  5  6+        """+        return self._clip_with_one_bound(threshold, method=self.ge,+                                         axis=axis, inplace=inplace)++    def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True,+                group_keys=True, squeeze=False, observed=False, **kwargs):+        """+        Group series using mapper (dict or key function, apply given function+        to group, return result as series) or by a series of columns.++        Parameters+        ----------+        by : mapping, function, label, or list of labels+            Used to determine the groups for the groupby.+            If ``by`` is a function, it's called on each value of the object's+            index. If a dict or Series is passed, the Series or dict VALUES+            will be used to determine the groups (the Series' values are first+            aligned; see ``.align()`` method). If an ndarray is passed, the+            values are used as-is determine the groups. A label or list of+            labels may be passed to group by the columns in ``self``. Notice+            that a tuple is interpreted a (single) key.+        axis : int, default 0+        level : int, level name, or sequence of such, default None+            If the axis is a MultiIndex (hierarchical), group by a particular+            level or levels+        as_index : boolean, default True+            For aggregated output, return object with group labels as the+            index. Only relevant for DataFrame input. as_index=False is+            effectively "SQL-style" grouped output+        sort : boolean, default True+            Sort group keys. Get better performance by turning this off.+            Note this does not influence the order of observations within each+            group.  groupby preserves the order of rows within each group.+        group_keys : boolean, default True+            When calling apply, add group keys to index to identify pieces+        squeeze : boolean, default False+            reduce the dimensionality of the return type if possible,+            otherwise return a consistent type+        observed : boolean, default False+            This only applies if any of the groupers are Categoricals+            If True: only show observed values for categorical groupers.+            If False: show all values for categorical groupers.++            .. versionadded:: 0.23.0++        Returns+        -------+        GroupBy object++        Examples+        --------+        DataFrame results++        >>> data.groupby(func, axis=0).mean()+        >>> data.groupby(['col1', 'col2'])['col3'].mean()++        DataFrame with hierarchical index++        >>> data.groupby(['col1', 'col2']).mean()++        Notes+        -----+        See the `user guide+        <http://pandas.pydata.org/pandas-docs/stable/groupby.html>`_ for more.++        See also+        --------+        resample : Convenience method for frequency conversion and resampling+            of time series.+        """+        from pandas.core.groupby.groupby import groupby++        if level is None and by is None:+            raise TypeError("You have to supply one of 'by' and 'level'")+        axis = self._get_axis_number(axis)+        return groupby(self, by=by, axis=axis, level=level, as_index=as_index,+                       sort=sort, group_keys=group_keys, squeeze=squeeze,+                       observed=observed, **kwargs)++    def asfreq(self, freq, method=None, how=None, normalize=False,+               fill_value=None):+        """+        Convert TimeSeries to specified frequency.++        Optionally provide filling method to pad/backfill missing values.++        Returns the original data conformed to a new index with the specified+        frequency. ``resample`` is more appropriate if an operation, such as+        summarization, is necessary to represent the data at the new frequency.++        Parameters+        ----------+        freq : DateOffset object, or string+        method : {'backfill'/'bfill', 'pad'/'ffill'}, default None+            Method to use for filling holes in reindexed Series (note this+            does not fill NaNs that already were present):++            * 'pad' / 'ffill': propagate last valid observation forward to next+              valid+            * 'backfill' / 'bfill': use NEXT valid observation to fill+        how : {'start', 'end'}, default end+            For PeriodIndex only, see PeriodIndex.asfreq+        normalize : bool, default False+            Whether to reset output index to midnight+        fill_value: scalar, optional+            Value to use for missing values, applied during upsampling (note+            this does not fill NaNs that already were present).++            .. versionadded:: 0.20.0++        Returns+        -------+        converted : same type as caller++        Examples+        --------++        Start by creating a series with 4 one minute timestamps.++        >>> index = pd.date_range('1/1/2000', periods=4, freq='T')+        >>> series = pd.Series([0.0, None, 2.0, 3.0], index=index)+        >>> df = pd.DataFrame({'s':series})+        >>> df+                               s+        2000-01-01 00:00:00    0.0+        2000-01-01 00:01:00    NaN+        2000-01-01 00:02:00    2.0+        2000-01-01 00:03:00    3.0++        Upsample the series into 30 second bins.++        >>> df.asfreq(freq='30S')+                               s+        2000-01-01 00:00:00    0.0+        2000-01-01 00:00:30    NaN+        2000-01-01 00:01:00    NaN+        2000-01-01 00:01:30    NaN+        2000-01-01 00:02:00    2.0+        2000-01-01 00:02:30    NaN+        2000-01-01 00:03:00    3.0++        Upsample again, providing a ``fill value``.++        >>> df.asfreq(freq='30S', fill_value=9.0)+                               s+        2000-01-01 00:00:00    0.0+        2000-01-01 00:00:30    9.0+        2000-01-01 00:01:00    NaN+        2000-01-01 00:01:30    9.0+        2000-01-01 00:02:00    2.0+        2000-01-01 00:02:30    9.0+        2000-01-01 00:03:00    3.0++        Upsample again, providing a ``method``.++        >>> df.asfreq(freq='30S', method='bfill')+                               s+        2000-01-01 00:00:00    0.0+        2000-01-01 00:00:30    NaN+        2000-01-01 00:01:00    NaN+        2000-01-01 00:01:30    2.0+        2000-01-01 00:02:00    2.0+        2000-01-01 00:02:30    3.0+        2000-01-01 00:03:00    3.0++        See Also+        --------+        reindex++        Notes+        -----+        To learn more about the frequency strings, please see `this link+        <http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases>`__.+        """+        from pandas.core.resample import asfreq+        return asfreq(self, freq, method=method, how=how, normalize=normalize,+                      fill_value=fill_value)++    def at_time(self, time, asof=False):+        """+        Select values at particular time of day (e.g. 9:30AM).++        Raises+        ------+        TypeError+            If the index is not  a :class:`DatetimeIndex`++        Parameters+        ----------+        time : datetime.time or string++        Returns+        -------+        values_at_time : same type as caller++        Examples+        --------+        >>> i = pd.date_range('2018-04-09', periods=4, freq='12H')+        >>> ts = pd.DataFrame({'A': [1,2,3,4]}, index=i)+        >>> ts+                             A+        2018-04-09 00:00:00  1+        2018-04-09 12:00:00  2+        2018-04-10 00:00:00  3+        2018-04-10 12:00:00  4++        >>> ts.at_time('12:00')+                             A+        2018-04-09 12:00:00  2+        2018-04-10 12:00:00  4++        See Also+        --------+        between_time : Select values between particular times of the day+        first : Select initial periods of time series based on a date offset+        last : Select final periods of time series based on a date offset+        DatetimeIndex.indexer_at_time : Get just the index locations for+            values at particular time of the day+        """+        try:+            indexer = self.index.indexer_at_time(time, asof=asof)+            return self._take(indexer)+        except AttributeError:+            raise TypeError('Index must be DatetimeIndex')++    def between_time(self, start_time, end_time, include_start=True,+                     include_end=True):+        """+        Select values between particular times of the day (e.g., 9:00-9:30 AM).++        By setting ``start_time`` to be later than ``end_time``,+        you can get the times that are *not* between the two times.++        Raises+        ------+        TypeError+            If the index is not  a :class:`DatetimeIndex`++        Parameters+        ----------+        start_time : datetime.time or string+        end_time : datetime.time or string+        include_start : boolean, default True+        include_end : boolean, default True++        Returns+        -------+        values_between_time : same type as caller++        Examples+        --------+        >>> i = pd.date_range('2018-04-09', periods=4, freq='1D20min')+        >>> ts = pd.DataFrame({'A': [1,2,3,4]}, index=i)+        >>> ts+                             A+        2018-04-09 00:00:00  1+        2018-04-10 00:20:00  2+        2018-04-11 00:40:00  3+        2018-04-12 01:00:00  4++        >>> ts.between_time('0:15', '0:45')+                             A+        2018-04-10 00:20:00  2+        2018-04-11 00:40:00  3++        You get the times that are *not* between two times by setting+        ``start_time`` later than ``end_time``:++        >>> ts.between_time('0:45', '0:15')+                             A+        2018-04-09 00:00:00  1+        2018-04-12 01:00:00  4++        See Also+        --------+        at_time : Select values at a particular time of the day+        first : Select initial periods of time series based on a date offset+        last : Select final periods of time series based on a date offset+        DatetimeIndex.indexer_between_time : Get just the index locations for+            values between particular times of the day+        """+        try:+            indexer = self.index.indexer_between_time(+                start_time, end_time, include_start=include_start,+                include_end=include_end)+            return self._take(indexer)+        except AttributeError:+            raise TypeError('Index must be DatetimeIndex')++    def resample(self, rule, how=None, axis=0, fill_method=None, closed=None,+                 label=None, convention='start', kind=None, loffset=None,+                 limit=None, base=0, on=None, level=None):+        """+        Convenience method for frequency conversion and resampling of time+        series.  Object must have a datetime-like index (DatetimeIndex,+        PeriodIndex, or TimedeltaIndex), or pass datetime-like values+        to the on or level keyword.++        Parameters+        ----------+        rule : string+            the offset string or object representing target conversion+        axis : int, optional, default 0+        closed : {'right', 'left'}+            Which side of bin interval is closed. The default is 'left'+            for all frequency offsets except for 'M', 'A', 'Q', 'BM',+            'BA', 'BQ', and 'W' which all have a default of 'right'.+        label : {'right', 'left'}+            Which bin edge label to label bucket with. The default is 'left'+            for all frequency offsets except for 'M', 'A', 'Q', 'BM',+            'BA', 'BQ', and 'W' which all have a default of 'right'.+        convention : {'start', 'end', 's', 'e'}+            For PeriodIndex only, controls whether to use the start or end of+            `rule`+        kind: {'timestamp', 'period'}, optional+            Pass 'timestamp' to convert the resulting index to a+            ``DateTimeIndex`` or 'period' to convert it to a ``PeriodIndex``.+            By default the input representation is retained.+        loffset : timedelta+            Adjust the resampled time labels+        base : int, default 0+            For frequencies that evenly subdivide 1 day, the "origin" of the+            aggregated intervals. For example, for '5min' frequency, base could+            range from 0 through 4. Defaults to 0+        on : string, optional+            For a DataFrame, column to use instead of index for resampling.+            Column must be datetime-like.++            .. versionadded:: 0.19.0++        level : string or int, optional+            For a MultiIndex, level (name or number) to use for+            resampling.  Level must be datetime-like.++            .. versionadded:: 0.19.0++        Returns+        -------+        Resampler object++        Notes+        -----+        See the `user guide+        <http://pandas.pydata.org/pandas-docs/stable/timeseries.html#resampling>`_+        for more.++        To learn more about the offset strings, please see `this link+        <http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases>`__.++        Examples+        --------++        Start by creating a series with 9 one minute timestamps.++        >>> index = pd.date_range('1/1/2000', periods=9, freq='T')+        >>> series = pd.Series(range(9), index=index)+        >>> series+        2000-01-01 00:00:00    0+        2000-01-01 00:01:00    1+        2000-01-01 00:02:00    2+        2000-01-01 00:03:00    3+        2000-01-01 00:04:00    4+        2000-01-01 00:05:00    5+        2000-01-01 00:06:00    6+        2000-01-01 00:07:00    7+        2000-01-01 00:08:00    8+        Freq: T, dtype: int64++        Downsample the series into 3 minute bins and sum the values+        of the timestamps falling into a bin.++        >>> series.resample('3T').sum()+        2000-01-01 00:00:00     3+        2000-01-01 00:03:00    12+        2000-01-01 00:06:00    21+        Freq: 3T, dtype: int64++        Downsample the series into 3 minute bins as above, but label each+        bin using the right edge instead of the left. Please note that the+        value in the bucket used as the label is not included in the bucket,+        which it labels. For example, in the original series the+        bucket ``2000-01-01 00:03:00`` contains the value 3, but the summed+        value in the resampled bucket with the label ``2000-01-01 00:03:00``+        does not include 3 (if it did, the summed value would be 6, not 3).+        To include this value close the right side of the bin interval as+        illustrated in the example below this one.++        >>> series.resample('3T', label='right').sum()+        2000-01-01 00:03:00     3+        2000-01-01 00:06:00    12+        2000-01-01 00:09:00    21+        Freq: 3T, dtype: int64++        Downsample the series into 3 minute bins as above, but close the right+        side of the bin interval.++        >>> series.resample('3T', label='right', closed='right').sum()+        2000-01-01 00:00:00     0+        2000-01-01 00:03:00     6+        2000-01-01 00:06:00    15+        2000-01-01 00:09:00    15+        Freq: 3T, dtype: int64++        Upsample the series into 30 second bins.++        >>> series.resample('30S').asfreq()[0:5] #select first 5 rows+        2000-01-01 00:00:00   0.0+        2000-01-01 00:00:30   NaN+        2000-01-01 00:01:00   1.0+        2000-01-01 00:01:30   NaN+        2000-01-01 00:02:00   2.0+        Freq: 30S, dtype: float64++        Upsample the series into 30 second bins and fill the ``NaN``+        values using the ``pad`` method.++        >>> series.resample('30S').pad()[0:5]+        2000-01-01 00:00:00    0+        2000-01-01 00:00:30    0+        2000-01-01 00:01:00    1+        2000-01-01 00:01:30    1+        2000-01-01 00:02:00    2+        Freq: 30S, dtype: int64++        Upsample the series into 30 second bins and fill the+        ``NaN`` values using the ``bfill`` method.++        >>> series.resample('30S').bfill()[0:5]+        2000-01-01 00:00:00    0+        2000-01-01 00:00:30    1+        2000-01-01 00:01:00    1+        2000-01-01 00:01:30    2+        2000-01-01 00:02:00    2+        Freq: 30S, dtype: int64++        Pass a custom function via ``apply``++        >>> def custom_resampler(array_like):+        ...     return np.sum(array_like)+5++        >>> series.resample('3T').apply(custom_resampler)+        2000-01-01 00:00:00     8+        2000-01-01 00:03:00    17+        2000-01-01 00:06:00    26+        Freq: 3T, dtype: int64++        For a Series with a PeriodIndex, the keyword `convention` can be+        used to control whether to use the start or end of `rule`.++        >>> s = pd.Series([1, 2], index=pd.period_range('2012-01-01',+                                                        freq='A',+                                                        periods=2))+        >>> s+        2012    1+        2013    2+        Freq: A-DEC, dtype: int64++        Resample by month using 'start' `convention`. Values are assigned to+        the first month of the period.++        >>> s.resample('M', convention='start').asfreq().head()+        2012-01    1.0+        2012-02    NaN+        2012-03    NaN+        2012-04    NaN+        2012-05    NaN+        Freq: M, dtype: float64++        Resample by month using 'end' `convention`. Values are assigned to+        the last month of the period.++        >>> s.resample('M', convention='end').asfreq()+        2012-12    1.0+        2013-01    NaN+        2013-02    NaN+        2013-03    NaN+        2013-04    NaN+        2013-05    NaN+        2013-06    NaN+        2013-07    NaN+        2013-08    NaN+        2013-09    NaN+        2013-10    NaN+        2013-11    NaN+        2013-12    2.0+        Freq: M, dtype: float64++        For DataFrame objects, the keyword ``on`` can be used to specify the+        column instead of the index for resampling.++        >>> df = pd.DataFrame(data=9*[range(4)], columns=['a', 'b', 'c', 'd'])+        >>> df['time'] = pd.date_range('1/1/2000', periods=9, freq='T')+        >>> df.resample('3T', on='time').sum()+                             a  b  c  d+        time+        2000-01-01 00:00:00  0  3  6  9+        2000-01-01 00:03:00  0  3  6  9+        2000-01-01 00:06:00  0  3  6  9++        For a DataFrame with MultiIndex, the keyword ``level`` can be used to+        specify on level the resampling needs to take place.++        >>> time = pd.date_range('1/1/2000', periods=5, freq='T')+        >>> df2 = pd.DataFrame(data=10*[range(4)],+                               columns=['a', 'b', 'c', 'd'],+                               index=pd.MultiIndex.from_product([time, [1, 2]])+                               )+        >>> df2.resample('3T', level=0).sum()+                             a  b   c   d+        2000-01-01 00:00:00  0  6  12  18+        2000-01-01 00:03:00  0  4   8  12++        See also+        --------+        groupby : Group by mapping, function, label, or list of labels.+        """+        from pandas.core.resample import (resample,+                                          _maybe_process_deprecations)+        axis = self._get_axis_number(axis)+        r = resample(self, freq=rule, label=label, closed=closed,+                     axis=axis, kind=kind, loffset=loffset,+                     convention=convention,+                     base=base, key=on, level=level)+        return _maybe_process_deprecations(r,+                                           how=how,+                                           fill_method=fill_method,+                                           limit=limit)++    def first(self, offset):+        """+        Convenience method for subsetting initial periods of time series data+        based on a date offset.++        Raises+        ------+        TypeError+            If the index is not  a :class:`DatetimeIndex`++        Parameters+        ----------+        offset : string, DateOffset, dateutil.relativedelta++        Examples+        --------+        >>> i = pd.date_range('2018-04-09', periods=4, freq='2D')+        >>> ts = pd.DataFrame({'A': [1,2,3,4]}, index=i)+        >>> ts+                    A+        2018-04-09  1+        2018-04-11  2+        2018-04-13  3+        2018-04-15  4++        Get the rows for the first 3 days:++        >>> ts.first('3D')+                    A+        2018-04-09  1+        2018-04-11  2++        Notice the data for 3 first calender days were returned, not the first+        3 days observed in the dataset, and therefore data for 2018-04-13 was+        not returned.++        Returns+        -------+        subset : same type as caller++        See Also+        --------+        last : Select final periods of time series based on a date offset+        at_time : Select values at a particular time of the day+        between_time : Select values between particular times of the day+        """+        if not isinstance(self.index, DatetimeIndex):+            raise TypeError("'first' only supports a DatetimeIndex index")++        if len(self.index) == 0:+            return self++        offset = to_offset(offset)+        end_date = end = self.index[0] + offset++        # Tick-like, e.g. 3 weeks+        if not offset.isAnchored() and hasattr(offset, '_inc'):+            if end_date in self.index:+                end = self.index.searchsorted(end_date, side='left')+                return self.iloc[:end]++        return self.loc[:end]++    def last(self, offset):+        """+        Convenience method for subsetting final periods of time series data+        based on a date offset.++        Raises+        ------+        TypeError+            If the index is not  a :class:`DatetimeIndex`++        Parameters+        ----------+        offset : string, DateOffset, dateutil.relativedelta++        Examples+        --------+        >>> i = pd.date_range('2018-04-09', periods=4, freq='2D')+        >>> ts = pd.DataFrame({'A': [1,2,3,4]}, index=i)+        >>> ts+                    A+        2018-04-09  1+        2018-04-11  2+        2018-04-13  3+        2018-04-15  4++        Get the rows for the last 3 days:++        >>> ts.last('3D')+                    A+        2018-04-13  3+        2018-04-15  4++        Notice the data for 3 last calender days were returned, not the last+        3 observed days in the dataset, and therefore data for 2018-04-11 was+        not returned.++        Returns+        -------+        subset : same type as caller++        See Also+        --------+        first : Select initial periods of time series based on a date offset+        at_time : Select values at a particular time of the day+        between_time : Select values between particular times of the day+        """+        if not isinstance(self.index, DatetimeIndex):+            raise TypeError("'last' only supports a DatetimeIndex index")++        if len(self.index) == 0:+            return self++        offset = to_offset(offset)++        start_date = self.index[-1] - offset+        start = self.index.searchsorted(start_date, side='right')+        return self.iloc[start:]++    def rank(self, axis=0, method='average', numeric_only=None,+             na_option='keep', ascending=True, pct=False):+        """+        Compute numerical data ranks (1 through n) along axis. Equal values are+        assigned a rank that is the average of the ranks of those values++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns'}, default 0+            index to direct ranking+        method : {'average', 'min', 'max', 'first', 'dense'}+            * average: average rank of group+            * min: lowest rank in group+            * max: highest rank in group+            * first: ranks assigned in order they appear in the array+            * dense: like 'min', but rank always increases by 1 between groups+        numeric_only : boolean, default None+            Include only float, int, boolean data. Valid only for DataFrame or+            Panel objects+        na_option : {'keep', 'top', 'bottom'}+            * keep: leave NA values where they are+            * top: smallest rank if ascending+            * bottom: smallest rank if descending+        ascending : boolean, default True+            False for ranks by high (1) to low (N)+        pct : boolean, default False+            Computes percentage rank of data++        Returns+        -------+        ranks : same type as caller+        """+        axis = self._get_axis_number(axis)++        if self.ndim > 2:+            msg = "rank does not make sense when ndim > 2"+            raise NotImplementedError(msg)++        if na_option not in {'keep', 'top', 'bottom'}:+            msg = "na_option must be one of 'keep', 'top', or 'bottom'"+            raise ValueError(msg)++        def ranker(data):+            ranks = algos.rank(data.values, axis=axis, method=method,+                               ascending=ascending, na_option=na_option,+                               pct=pct)+            ranks = self._constructor(ranks, **data._construct_axes_dict())+            return ranks.__finalize__(self)++        # if numeric_only is None, and we can't get anything, we try with+        # numeric_only=True+        if numeric_only is None:+            try:+                return ranker(self)+            except TypeError:+                numeric_only = True++        if numeric_only:+            data = self._get_numeric_data()+        else:+            data = self++        return ranker(data)++    _shared_docs['align'] = ("""+        Align two objects on their axes with the+        specified join method for each axis Index++        Parameters+        ----------+        other : DataFrame or Series+        join : {'outer', 'inner', 'left', 'right'}, default 'outer'+        axis : allowed axis of the other object, default None+            Align on index (0), columns (1), or both (None)+        level : int or level name, default None+            Broadcast across a level, matching Index values on the+            passed MultiIndex level+        copy : boolean, default True+            Always returns new objects. If copy=False and no reindexing is+            required then original objects are returned.+        fill_value : scalar, default np.NaN+            Value to use for missing values. Defaults to NaN, but can be any+            "compatible" value+        method : str, default None+        limit : int, default None+        fill_axis : %(axes_single_arg)s, default 0+            Filling axis, method and limit+        broadcast_axis : %(axes_single_arg)s, default None+            Broadcast values along this axis, if aligning two objects of+            different dimensions++        Returns+        -------+        (left, right) : (%(klass)s, type of other)+            Aligned objects+        """)++    @Appender(_shared_docs['align'] % _shared_doc_kwargs)+    def align(self, other, join='outer', axis=None, level=None, copy=True,+              fill_value=None, method=None, limit=None, fill_axis=0,+              broadcast_axis=None):+        from pandas import DataFrame, Series+        method = missing.clean_fill_method(method)++        if broadcast_axis == 1 and self.ndim != other.ndim:+            if isinstance(self, Series):+                # this means other is a DataFrame, and we need to broadcast+                # self+                cons = self._constructor_expanddim+                df = cons({c: self for c in other.columns},+                          **other._construct_axes_dict())+                return df._align_frame(other, join=join, axis=axis,+                                       level=level, copy=copy,+                                       fill_value=fill_value, method=method,+                                       limit=limit, fill_axis=fill_axis)+            elif isinstance(other, Series):+                # this means self is a DataFrame, and we need to broadcast+                # other+                cons = other._constructor_expanddim+                df = cons({c: other for c in self.columns},+                          **self._construct_axes_dict())+                return self._align_frame(df, join=join, axis=axis, level=level,+                                         copy=copy, fill_value=fill_value,+                                         method=method, limit=limit,+                                         fill_axis=fill_axis)++        if axis is not None:+            axis = self._get_axis_number(axis)+        if isinstance(other, DataFrame):+            return self._align_frame(other, join=join, axis=axis, level=level,+                                     copy=copy, fill_value=fill_value,+                                     method=method, limit=limit,+                                     fill_axis=fill_axis)+        elif isinstance(other, Series):+            return self._align_series(other, join=join, axis=axis, level=level,+                                      copy=copy, fill_value=fill_value,+                                      method=method, limit=limit,+                                      fill_axis=fill_axis)+        else:  # pragma: no cover+            raise TypeError('unsupported type: %s' % type(other))++    def _align_frame(self, other, join='outer', axis=None, level=None,+                     copy=True, fill_value=None, method=None, limit=None,+                     fill_axis=0):+        # defaults+        join_index, join_columns = None, None+        ilidx, iridx = None, None+        clidx, cridx = None, None++        is_series = isinstance(self, ABCSeries)++        if axis is None or axis == 0:+            if not self.index.equals(other.index):+                join_index, ilidx, iridx = self.index.join(+                    other.index, how=join, level=level, return_indexers=True)++        if axis is None or axis == 1:+            if not is_series and not self.columns.equals(other.columns):+                join_columns, clidx, cridx = self.columns.join(+                    other.columns, how=join, level=level, return_indexers=True)++        if is_series:+            reindexers = {0: [join_index, ilidx]}+        else:+            reindexers = {0: [join_index, ilidx], 1: [join_columns, clidx]}++        left = self._reindex_with_indexers(reindexers, copy=copy,+                                           fill_value=fill_value,+                                           allow_dups=True)+        # other must be always DataFrame+        right = other._reindex_with_indexers({0: [join_index, iridx],+                                              1: [join_columns, cridx]},+                                             copy=copy, fill_value=fill_value,+                                             allow_dups=True)++        if method is not None:+            left = left.fillna(axis=fill_axis, method=method, limit=limit)+            right = right.fillna(axis=fill_axis, method=method, limit=limit)++        # if DatetimeIndex have different tz, convert to UTC+        if is_datetime64tz_dtype(left.index):+            if left.index.tz != right.index.tz:+                if join_index is not None:+                    left.index = join_index+                    right.index = join_index++        return left.__finalize__(self), right.__finalize__(other)++    def _align_series(self, other, join='outer', axis=None, level=None,+                      copy=True, fill_value=None, method=None, limit=None,+                      fill_axis=0):++        is_series = isinstance(self, ABCSeries)++        # series/series compat, other must always be a Series+        if is_series:+            if axis:+                raise ValueError('cannot align series to a series other than '+                                 'axis 0')++            # equal+            if self.index.equals(other.index):+                join_index, lidx, ridx = None, None, None+            else:+                join_index, lidx, ridx = self.index.join(other.index, how=join,+                                                         level=level,+                                                         return_indexers=True)++            left = self._reindex_indexer(join_index, lidx, copy)+            right = other._reindex_indexer(join_index, ridx, copy)++        else:+            # one has > 1 ndim+            fdata = self._data+            if axis == 0:+                join_index = self.index+                lidx, ridx = None, None+                if not self.index.equals(other.index):+                    join_index, lidx, ridx = self.index.join(+                        other.index, how=join, level=level,+                        return_indexers=True)++                if lidx is not None:+                    fdata = fdata.reindex_indexer(join_index, lidx, axis=1)++            elif axis == 1:+                join_index = self.columns+                lidx, ridx = None, None+                if not self.columns.equals(other.index):+                    join_index, lidx, ridx = self.columns.join(+                        other.index, how=join, level=level,+                        return_indexers=True)++                if lidx is not None:+                    fdata = fdata.reindex_indexer(join_index, lidx, axis=0)+            else:+                raise ValueError('Must specify axis=0 or 1')++            if copy and fdata is self._data:+                fdata = fdata.copy()++            left = self._constructor(fdata)++            if ridx is None:+                right = other+            else:+                right = other.reindex(join_index, level=level)++        # fill+        fill_na = notna(fill_value) or (method is not None)+        if fill_na:+            left = left.fillna(fill_value, method=method, limit=limit,+                               axis=fill_axis)+            right = right.fillna(fill_value, method=method, limit=limit)++        # if DatetimeIndex have different tz, convert to UTC+        if is_series or (not is_series and axis == 0):+            if is_datetime64tz_dtype(left.index):+                if left.index.tz != right.index.tz:+                    if join_index is not None:+                        left.index = join_index+                        right.index = join_index++        return left.__finalize__(self), right.__finalize__(other)++    def _where(self, cond, other=np.nan, inplace=False, axis=None, level=None,+               errors='raise', try_cast=False):+        """+        Equivalent to public method `where`, except that `other` is not+        applied as a function even if callable. Used in __setitem__.+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')++        # align the cond to same shape as myself+        cond = com.apply_if_callable(cond, self)+        if isinstance(cond, NDFrame):+            cond, _ = cond.align(self, join='right', broadcast_axis=1)+        else:+            if not hasattr(cond, 'shape'):+                cond = np.asanyarray(cond)+            if cond.shape != self.shape:+                raise ValueError('Array conditional must be same shape as '+                                 'self')+            cond = self._constructor(cond, **self._construct_axes_dict())++        # make sure we are boolean+        fill_value = True if inplace else False+        cond = cond.fillna(fill_value)++        msg = "Boolean array expected for the condition, not {dtype}"++        if not isinstance(cond, pd.DataFrame):+            # This is a single-dimensional object.+            if not is_bool_dtype(cond):+                raise ValueError(msg.format(dtype=cond.dtype))+        else:+            for dt in cond.dtypes:+                if not is_bool_dtype(dt):+                    raise ValueError(msg.format(dtype=dt))++        cond = -cond if inplace else cond++        # try to align with other+        try_quick = True+        if hasattr(other, 'align'):++            # align with me+            if other.ndim <= self.ndim:++                _, other = self.align(other, join='left', axis=axis,+                                      level=level, fill_value=np.nan)++                # if we are NOT aligned, raise as we cannot where index+                if (axis is None and+                        not all(other._get_axis(i).equals(ax)+                                for i, ax in enumerate(self.axes))):+                    raise InvalidIndexError++            # slice me out of the other+            else:+                raise NotImplementedError("cannot align with a higher "+                                          "dimensional NDFrame")++        if isinstance(other, np.ndarray):++            if other.shape != self.shape:++                if self.ndim == 1:++                    icond = cond.values++                    # GH 2745 / GH 4192+                    # treat like a scalar+                    if len(other) == 1:+                        other = np.array(other[0])++                    # GH 3235+                    # match True cond to other+                    elif len(cond[icond]) == len(other):++                        # try to not change dtype at first (if try_quick)+                        if try_quick:++                            try:+                                new_other = com.values_from_object(self)+                                new_other = new_other.copy()+                                new_other[icond] = other+                                other = new_other+                            except Exception:+                                try_quick = False++                        # let's create a new (if we failed at the above+                        # or not try_quick+                        if not try_quick:++                            dtype, fill_value = maybe_promote(other.dtype)+                            new_other = np.empty(len(icond), dtype=dtype)+                            new_other.fill(fill_value)+                            maybe_upcast_putmask(new_other, icond, other)+                            other = new_other++                    else:+                        raise ValueError('Length of replacements must equal '+                                         'series length')++                else:+                    raise ValueError('other must be the same shape as self '+                                     'when an ndarray')++            # we are the same shape, so create an actual object for alignment+            else:+                other = self._constructor(other, **self._construct_axes_dict())++        if axis is None:+            axis = 0++        if self.ndim == getattr(other, 'ndim', 0):+            align = True+        else:+            align = (self._get_axis_number(axis) == 1)++        block_axis = self._get_block_manager_axis(axis)++        if inplace:+            # we may have different type blocks come out of putmask, so+            # reconstruct the block manager++            self._check_inplace_setting(other)+            new_data = self._data.putmask(mask=cond, new=other, align=align,+                                          inplace=True, axis=block_axis,+                                          transpose=self._AXIS_REVERSED)+            self._update_inplace(new_data)++        else:+            new_data = self._data.where(other=other, cond=cond, align=align,+                                        errors=errors,+                                        try_cast=try_cast, axis=block_axis,+                                        transpose=self._AXIS_REVERSED)++            return self._constructor(new_data).__finalize__(self)++    _shared_docs['where'] = ("""+        Replace values where the condition is %(cond_rev)s.++        Parameters+        ----------+        cond : boolean %(klass)s, array-like, or callable+            Where `cond` is %(cond)s, keep the original value. Where+            %(cond_rev)s, replace with corresponding value from `other`.+            If `cond` is callable, it is computed on the %(klass)s and+            should return boolean %(klass)s or array. The callable must+            not change input %(klass)s (though pandas doesn't check it).++            .. versionadded:: 0.18.1+                A callable can be used as cond.++        other : scalar, %(klass)s, or callable+            Entries where `cond` is %(cond_rev)s are replaced with+            corresponding value from `other`.+            If other is callable, it is computed on the %(klass)s and+            should return scalar or %(klass)s. The callable must not+            change input %(klass)s (though pandas doesn't check it).++            .. versionadded:: 0.18.1+                A callable can be used as other.++        inplace : boolean, default False+            Whether to perform the operation in place on the data.+        axis : int, default None+            Alignment axis if needed.+        level : int, default None+            Alignment level if needed.+        errors : str, {'raise', 'ignore'}, default `raise`+            Note that currently this parameter won't affect+            the results and will always coerce to a suitable dtype.++            - `raise` : allow exceptions to be raised.+            - `ignore` : suppress exceptions. On error return original object.++        try_cast : boolean, default False+            Try to cast the result back to the input type (if possible).+        raise_on_error : boolean, default True+            Whether to raise on invalid data types (e.g. trying to where on+            strings).++            .. deprecated:: 0.21.0++               Use `errors`.++        Returns+        -------+        wh : same type as caller++        Notes+        -----+        The %(name)s method is an application of the if-then idiom. For each+        element in the calling DataFrame, if ``cond`` is ``%(cond)s`` the+        element is used; otherwise the corresponding element from the DataFrame+        ``other`` is used.++        The signature for :func:`DataFrame.where` differs from+        :func:`numpy.where`. Roughly ``df1.where(m, df2)`` is equivalent to+        ``np.where(m, df1, df2)``.++        For further details and examples see the ``%(name)s`` documentation in+        :ref:`indexing <indexing.where_mask>`.++        See Also+        --------+        :func:`DataFrame.%(name_other)s` : Return an object of same shape as+            self++        Examples+        --------+        >>> s = pd.Series(range(5))+        >>> s.where(s > 0)+        0    NaN+        1    1.0+        2    2.0+        3    3.0+        4    4.0+        dtype: float64++        >>> s.mask(s > 0)+        0    0.0+        1    NaN+        2    NaN+        3    NaN+        4    NaN+        dtype: float64++        >>> s.where(s > 1, 10)+        0    10+        1    10+        2    2+        3    3+        4    4+        dtype: int64++        >>> df = pd.DataFrame(np.arange(10).reshape(-1, 2), columns=['A', 'B'])+        >>> m = df %% 3 == 0+        >>> df.where(m, -df)+           A  B+        0  0 -1+        1 -2  3+        2 -4 -5+        3  6 -7+        4 -8  9+        >>> df.where(m, -df) == np.where(m, df, -df)+              A     B+        0  True  True+        1  True  True+        2  True  True+        3  True  True+        4  True  True+        >>> df.where(m, -df) == df.mask(~m, -df)+              A     B+        0  True  True+        1  True  True+        2  True  True+        3  True  True+        4  True  True+        """)++    @Appender(_shared_docs['where'] % dict(_shared_doc_kwargs, cond="True",+                                           cond_rev="False", name='where',+                                           name_other='mask'))+    def where(self, cond, other=np.nan, inplace=False, axis=None, level=None,+              errors='raise', try_cast=False, raise_on_error=None):++        if raise_on_error is not None:+            warnings.warn(+                "raise_on_error is deprecated in "+                "favor of errors='raise|ignore'",+                FutureWarning, stacklevel=2)++            if raise_on_error:+                errors = 'raise'+            else:+                errors = 'ignore'++        other = com.apply_if_callable(other, self)+        return self._where(cond, other, inplace, axis, level,+                           errors=errors, try_cast=try_cast)++    @Appender(_shared_docs['where'] % dict(_shared_doc_kwargs, cond="False",+                                           cond_rev="True", name='mask',+                                           name_other='where'))+    def mask(self, cond, other=np.nan, inplace=False, axis=None, level=None,+             errors='raise', try_cast=False, raise_on_error=None):++        if raise_on_error is not None:+            warnings.warn(+                "raise_on_error is deprecated in "+                "favor of errors='raise|ignore'",+                FutureWarning, stacklevel=2)++            if raise_on_error:+                errors = 'raise'+            else:+                errors = 'ignore'++        inplace = validate_bool_kwarg(inplace, 'inplace')+        cond = com.apply_if_callable(cond, self)++        # see gh-21891+        if not hasattr(cond, "__invert__"):+            cond = np.array(cond)++        return self.where(~cond, other=other, inplace=inplace, axis=axis,+                          level=level, try_cast=try_cast,+                          errors=errors)++    _shared_docs['shift'] = ("""+        Shift index by desired number of periods with an optional time freq++        Parameters+        ----------+        periods : int+            Number of periods to move, can be positive or negative+        freq : DateOffset, timedelta, or time rule string, optional+            Increment to use from the tseries module or time rule (e.g. 'EOM').+            See Notes.+        axis : %(axes_single_arg)s++        Notes+        -----+        If freq is specified then the index values are shifted but the data+        is not realigned. That is, use freq if you would like to extend the+        index when shifting and preserve the original data.++        Returns+        -------+        shifted : %(klass)s+    """)++    @Appender(_shared_docs['shift'] % _shared_doc_kwargs)+    def shift(self, periods=1, freq=None, axis=0):+        if periods == 0:+            return self++        block_axis = self._get_block_manager_axis(axis)+        if freq is None:+            new_data = self._data.shift(periods=periods, axis=block_axis)+        else:+            return self.tshift(periods, freq)++        return self._constructor(new_data).__finalize__(self)++    def slice_shift(self, periods=1, axis=0):+        """+        Equivalent to `shift` without copying data. The shifted data will+        not include the dropped periods and the shifted axis will be smaller+        than the original.++        Parameters+        ----------+        periods : int+            Number of periods to move, can be positive or negative++        Notes+        -----+        While the `slice_shift` is faster than `shift`, you may pay for it+        later during alignment.++        Returns+        -------+        shifted : same type as caller+        """+        if periods == 0:+            return self++        if periods > 0:+            vslicer = slice(None, -periods)+            islicer = slice(periods, None)+        else:+            vslicer = slice(-periods, None)+            islicer = slice(None, periods)++        new_obj = self._slice(vslicer, axis=axis)+        shifted_axis = self._get_axis(axis)[islicer]+        new_obj.set_axis(shifted_axis, axis=axis, inplace=True)++        return new_obj.__finalize__(self)++    def tshift(self, periods=1, freq=None, axis=0):+        """+        Shift the time index, using the index's frequency if available.++        Parameters+        ----------+        periods : int+            Number of periods to move, can be positive or negative+        freq : DateOffset, timedelta, or time rule string, default None+            Increment to use from the tseries module or time rule (e.g. 'EOM')+        axis : int or basestring+            Corresponds to the axis that contains the Index++        Notes+        -----+        If freq is not specified then tries to use the freq or inferred_freq+        attributes of the index. If neither of those attributes exist, a+        ValueError is thrown++        Returns+        -------+        shifted : NDFrame+        """++        index = self._get_axis(axis)+        if freq is None:+            freq = getattr(index, 'freq', None)++        if freq is None:+            freq = getattr(index, 'inferred_freq', None)++        if freq is None:+            msg = 'Freq was not given and was not set in the index'+            raise ValueError(msg)++        if periods == 0:+            return self++        if isinstance(freq, string_types):+            freq = to_offset(freq)++        block_axis = self._get_block_manager_axis(axis)+        if isinstance(index, PeriodIndex):+            orig_freq = to_offset(index.freq)+            if freq == orig_freq:+                new_data = self._data.copy()+                new_data.axes[block_axis] = index.shift(periods)+            else:+                msg = ('Given freq %s does not match PeriodIndex freq %s' %+                       (freq.rule_code, orig_freq.rule_code))+                raise ValueError(msg)+        else:+            new_data = self._data.copy()+            new_data.axes[block_axis] = index.shift(periods, freq)++        return self._constructor(new_data).__finalize__(self)++    def truncate(self, before=None, after=None, axis=None, copy=True):+        """+        Truncate a Series or DataFrame before and after some index value.++        This is a useful shorthand for boolean indexing based on index+        values above or below certain thresholds.++        Parameters+        ----------+        before : date, string, int+            Truncate all rows before this index value.+        after : date, string, int+            Truncate all rows after this index value.+        axis : {0 or 'index', 1 or 'columns'}, optional+            Axis to truncate. Truncates the index (rows) by default.+        copy : boolean, default is True,+            Return a copy of the truncated section.++        Returns+        -------+        type of caller+            The truncated Series or DataFrame.++        See Also+        --------+        DataFrame.loc : Select a subset of a DataFrame by label.+        DataFrame.iloc : Select a subset of a DataFrame by position.++        Notes+        -----+        If the index being truncated contains only datetime values,+        `before` and `after` may be specified as strings instead of+        Timestamps.++        Examples+        --------+        >>> df = pd.DataFrame({'A': ['a', 'b', 'c', 'd', 'e'],+        ...                    'B': ['f', 'g', 'h', 'i', 'j'],+        ...                    'C': ['k', 'l', 'm', 'n', 'o']},+        ...                    index=[1, 2, 3, 4, 5])+        >>> df+           A  B  C+        1  a  f  k+        2  b  g  l+        3  c  h  m+        4  d  i  n+        5  e  j  o++        >>> df.truncate(before=2, after=4)+           A  B  C+        2  b  g  l+        3  c  h  m+        4  d  i  n++        The columns of a DataFrame can be truncated.++        >>> df.truncate(before="A", after="B", axis="columns")+           A  B+        1  a  f+        2  b  g+        3  c  h+        4  d  i+        5  e  j++        For Series, only rows can be truncated.++        >>> df['A'].truncate(before=2, after=4)+        2    b+        3    c+        4    d+        Name: A, dtype: object++        The index values in ``truncate`` can be datetimes or string+        dates.++        >>> dates = pd.date_range('2016-01-01', '2016-02-01', freq='s')+        >>> df = pd.DataFrame(index=dates, data={'A': 1})+        >>> df.tail()+                             A+        2016-01-31 23:59:56  1+        2016-01-31 23:59:57  1+        2016-01-31 23:59:58  1+        2016-01-31 23:59:59  1+        2016-02-01 00:00:00  1++        >>> df.truncate(before=pd.Timestamp('2016-01-05'),+        ...             after=pd.Timestamp('2016-01-10')).tail()+                             A+        2016-01-09 23:59:56  1+        2016-01-09 23:59:57  1+        2016-01-09 23:59:58  1+        2016-01-09 23:59:59  1+        2016-01-10 00:00:00  1++        Because the index is a DatetimeIndex containing only dates, we can+        specify `before` and `after` as strings. They will be coerced to+        Timestamps before truncation.++        >>> df.truncate('2016-01-05', '2016-01-10').tail()+                             A+        2016-01-09 23:59:56  1+        2016-01-09 23:59:57  1+        2016-01-09 23:59:58  1+        2016-01-09 23:59:59  1+        2016-01-10 00:00:00  1++        Note that ``truncate`` assumes a 0 value for any unspecified time+        component (midnight). This differs from partial string slicing, which+        returns any partially matching dates.++        >>> df.loc['2016-01-05':'2016-01-10', :].tail()+                             A+        2016-01-10 23:59:55  1+        2016-01-10 23:59:56  1+        2016-01-10 23:59:57  1+        2016-01-10 23:59:58  1+        2016-01-10 23:59:59  1+        """++        if axis is None:+            axis = self._stat_axis_number+        axis = self._get_axis_number(axis)+        ax = self._get_axis(axis)++        # GH 17935+        # Check that index is sorted+        if not ax.is_monotonic_increasing and not ax.is_monotonic_decreasing:+            raise ValueError("truncate requires a sorted index")++        # if we have a date index, convert to dates, otherwise+        # treat like a slice+        if ax.is_all_dates:+            from pandas.core.tools.datetimes import to_datetime+            before = to_datetime(before)+            after = to_datetime(after)++        if before is not None and after is not None:+            if before > after:+                raise ValueError('Truncate: %s must be after %s' %+                                 (after, before))++        slicer = [slice(None, None)] * self._AXIS_LEN+        slicer[axis] = slice(before, after)+        result = self.loc[tuple(slicer)]++        if isinstance(ax, MultiIndex):+            setattr(result, self._get_axis_name(axis),+                    ax.truncate(before, after))++        if copy:+            result = result.copy()++        return result++    def tz_convert(self, tz, axis=0, level=None, copy=True):+        """+        Convert tz-aware axis to target time zone.++        Parameters+        ----------+        tz : string or pytz.timezone object+        axis : the axis to convert+        level : int, str, default None+            If axis ia a MultiIndex, convert a specific level. Otherwise+            must be None+        copy : boolean, default True+            Also make a copy of the underlying data++        Returns+        -------++        Raises+        ------+        TypeError+            If the axis is tz-naive.+        """+        axis = self._get_axis_number(axis)+        ax = self._get_axis(axis)++        def _tz_convert(ax, tz):+            if not hasattr(ax, 'tz_convert'):+                if len(ax) > 0:+                    ax_name = self._get_axis_name(axis)+                    raise TypeError('%s is not a valid DatetimeIndex or '+                                    'PeriodIndex' % ax_name)+                else:+                    ax = DatetimeIndex([], tz=tz)+            else:+                ax = ax.tz_convert(tz)+            return ax++        # if a level is given it must be a MultiIndex level or+        # equivalent to the axis name+        if isinstance(ax, MultiIndex):+            level = ax._get_level_number(level)+            new_level = _tz_convert(ax.levels[level], tz)+            ax = ax.set_levels(new_level, level=level)+        else:+            if level not in (None, 0, ax.name):+                raise ValueError("The level {0} is not valid".format(level))+            ax = _tz_convert(ax, tz)++        result = self._constructor(self._data, copy=copy)+        result.set_axis(ax, axis=axis, inplace=True)+        return result.__finalize__(self)++    def tz_localize(self, tz, axis=0, level=None, copy=True,+                    ambiguous='raise'):+        """+        Localize tz-naive TimeSeries to target time zone.++        Parameters+        ----------+        tz : string or pytz.timezone object+        axis : the axis to localize+        level : int, str, default None+            If axis ia a MultiIndex, localize a specific level. Otherwise+            must be None+        copy : boolean, default True+            Also make a copy of the underlying data+        ambiguous : 'infer', bool-ndarray, 'NaT', default 'raise'+            - 'infer' will attempt to infer fall dst-transition hours based on+              order+            - bool-ndarray where True signifies a DST time, False designates+              a non-DST time (note that this flag is only applicable for+              ambiguous times)+            - 'NaT' will return NaT where there are ambiguous times+            - 'raise' will raise an AmbiguousTimeError if there are ambiguous+              times++        Returns+        -------++        Raises+        ------+        TypeError+            If the TimeSeries is tz-aware and tz is not None.+        """+        axis = self._get_axis_number(axis)+        ax = self._get_axis(axis)++        def _tz_localize(ax, tz, ambiguous):+            if not hasattr(ax, 'tz_localize'):+                if len(ax) > 0:+                    ax_name = self._get_axis_name(axis)+                    raise TypeError('%s is not a valid DatetimeIndex or '+                                    'PeriodIndex' % ax_name)+                else:+                    ax = DatetimeIndex([], tz=tz)+            else:+                ax = ax.tz_localize(tz, ambiguous=ambiguous)+            return ax++        # if a level is given it must be a MultiIndex level or+        # equivalent to the axis name+        if isinstance(ax, MultiIndex):+            level = ax._get_level_number(level)+            new_level = _tz_localize(ax.levels[level], tz, ambiguous)+            ax = ax.set_levels(new_level, level=level)+        else:+            if level not in (None, 0, ax.name):+                raise ValueError("The level {0} is not valid".format(level))+            ax = _tz_localize(ax, tz, ambiguous)++        result = self._constructor(self._data, copy=copy)+        result.set_axis(ax, axis=axis, inplace=True)+        return result.__finalize__(self)++    # ----------------------------------------------------------------------+    # Numeric Methods+    def abs(self):+        """+        Return a Series/DataFrame with absolute numeric value of each element.++        This function only applies to elements that are all numeric.++        Returns+        -------+        abs+            Series/DataFrame containing the absolute value of each element.++        Notes+        -----+        For ``complex`` inputs, ``1.2 + 1j``, the absolute value is+        :math:`\\sqrt{ a^2 + b^2 }`.++        Examples+        --------+        Absolute numeric values in a Series.++        >>> s = pd.Series([-1.10, 2, -3.33, 4])+        >>> s.abs()+        0    1.10+        1    2.00+        2    3.33+        3    4.00+        dtype: float64++        Absolute numeric values in a Series with complex numbers.++        >>> s = pd.Series([1.2 + 1j])+        >>> s.abs()+        0    1.56205+        dtype: float64++        Absolute numeric values in a Series with a Timedelta element.++        >>> s = pd.Series([pd.Timedelta('1 days')])+        >>> s.abs()+        0   1 days+        dtype: timedelta64[ns]++        Select rows with data closest to certain value using argsort (from+        `StackOverflow <https://stackoverflow.com/a/17758115>`__).++        >>> df = pd.DataFrame({+        ...     'a': [4, 5, 6, 7],+        ...     'b': [10, 20, 30, 40],+        ...     'c': [100, 50, -30, -50]+        ... })+        >>> df+             a    b    c+        0    4   10  100+        1    5   20   50+        2    6   30  -30+        3    7   40  -50+        >>> df.loc[(df.c - 43).abs().argsort()]+             a    b    c+        1    5   20   50+        0    4   10  100+        2    6   30  -30+        3    7   40  -50++        See Also+        --------+        numpy.absolute : calculate the absolute value element-wise.+        """+        return np.abs(self)++    def describe(self, percentiles=None, include=None, exclude=None):+        """+        Generate descriptive statistics that summarize the central tendency,+        dispersion and shape of a dataset's distribution, excluding+        ``NaN`` values.++        Analyzes both numeric and object series, as well+        as ``DataFrame`` column sets of mixed data types. The output+        will vary depending on what is provided. Refer to the notes+        below for more detail.++        Parameters+        ----------+        percentiles : list-like of numbers, optional+            The percentiles to include in the output. All should+            fall between 0 and 1. The default is+            ``[.25, .5, .75]``, which returns the 25th, 50th, and+            75th percentiles.+        include : 'all', list-like of dtypes or None (default), optional+            A white list of data types to include in the result. Ignored+            for ``Series``. Here are the options:++            - 'all' : All columns of the input will be included in the output.+            - A list-like of dtypes : Limits the results to the+              provided data types.+              To limit the result to numeric types submit+              ``numpy.number``. To limit it instead to object columns submit+              the ``numpy.object`` data type. Strings+              can also be used in the style of+              ``select_dtypes`` (e.g. ``df.describe(include=['O'])``). To+              select pandas categorical columns, use ``'category'``+            - None (default) : The result will include all numeric columns.+        exclude : list-like of dtypes or None (default), optional,+            A black list of data types to omit from the result. Ignored+            for ``Series``. Here are the options:++            - A list-like of dtypes : Excludes the provided data types+              from the result. To exclude numeric types submit+              ``numpy.number``. To exclude object columns submit the data+              type ``numpy.object``. Strings can also be used in the style of+              ``select_dtypes`` (e.g. ``df.describe(include=['O'])``). To+              exclude pandas categorical columns, use ``'category'``+            - None (default) : The result will exclude nothing.++        Returns+        -------+        Series or DataFrame+            Summary statistics of the Series or Dataframe provided.++        See Also+        --------+        DataFrame.count: Count number of non-NA/null observations.+        DataFrame.max: Maximum of the values in the object.+        DataFrame.min: Minimum of the values in the object.+        DataFrame.mean: Mean of the values.+        DataFrame.std: Standard deviation of the obersvations.+        DataFrame.select_dtypes: Subset of a DataFrame including/excluding+            columns based on their dtype.++        Notes+        -----+        For numeric data, the result's index will include ``count``,+        ``mean``, ``std``, ``min``, ``max`` as well as lower, ``50`` and+        upper percentiles. By default the lower percentile is ``25`` and the+        upper percentile is ``75``. The ``50`` percentile is the+        same as the median.++        For object data (e.g. strings or timestamps), the result's index+        will include ``count``, ``unique``, ``top``, and ``freq``. The ``top``+        is the most common value. The ``freq`` is the most common value's+        frequency. Timestamps also include the ``first`` and ``last`` items.++        If multiple object values have the highest count, then the+        ``count`` and ``top`` results will be arbitrarily chosen from+        among those with the highest count.++        For mixed data types provided via a ``DataFrame``, the default is to+        return only an analysis of numeric columns. If the dataframe consists+        only of object and categorical data without any numeric columns, the+        default is to return an analysis of both the object and categorical+        columns. If ``include='all'`` is provided as an option, the result+        will include a union of attributes of each type.++        The `include` and `exclude` parameters can be used to limit+        which columns in a ``DataFrame`` are analyzed for the output.+        The parameters are ignored when analyzing a ``Series``.++        Examples+        --------+        Describing a numeric ``Series``.++        >>> s = pd.Series([1, 2, 3])+        >>> s.describe()+        count    3.0+        mean     2.0+        std      1.0+        min      1.0+        25%      1.5+        50%      2.0+        75%      2.5+        max      3.0+        dtype: float64++        Describing a categorical ``Series``.++        >>> s = pd.Series(['a', 'a', 'b', 'c'])+        >>> s.describe()+        count     4+        unique    3+        top       a+        freq      2+        dtype: object++        Describing a timestamp ``Series``.++        >>> s = pd.Series([+        ...   np.datetime64("2000-01-01"),+        ...   np.datetime64("2010-01-01"),+        ...   np.datetime64("2010-01-01")+        ... ])+        >>> s.describe()+        count                       3+        unique                      2+        top       2010-01-01 00:00:00+        freq                        2+        first     2000-01-01 00:00:00+        last      2010-01-01 00:00:00+        dtype: object++        Describing a ``DataFrame``. By default only numeric fields+        are returned.++        >>> df = pd.DataFrame({'categorical': pd.Categorical(['d','e','f']),+        ...                    'numeric': [1, 2, 3],+        ...                    'object': ['a', 'b', 'c']+        ...                   })+        >>> df.describe()+               numeric+        count      3.0+        mean       2.0+        std        1.0+        min        1.0+        25%        1.5+        50%        2.0+        75%        2.5+        max        3.0++        Describing all columns of a ``DataFrame`` regardless of data type.++        >>> df.describe(include='all')+                categorical  numeric object+        count            3      3.0      3+        unique           3      NaN      3+        top              f      NaN      c+        freq             1      NaN      1+        mean           NaN      2.0    NaN+        std            NaN      1.0    NaN+        min            NaN      1.0    NaN+        25%            NaN      1.5    NaN+        50%            NaN      2.0    NaN+        75%            NaN      2.5    NaN+        max            NaN      3.0    NaN++        Describing a column from a ``DataFrame`` by accessing it as+        an attribute.++        >>> df.numeric.describe()+        count    3.0+        mean     2.0+        std      1.0+        min      1.0+        25%      1.5+        50%      2.0+        75%      2.5+        max      3.0+        Name: numeric, dtype: float64++        Including only numeric columns in a ``DataFrame`` description.++        >>> df.describe(include=[np.number])+               numeric+        count      3.0+        mean       2.0+        std        1.0+        min        1.0+        25%        1.5+        50%        2.0+        75%        2.5+        max        3.0++        Including only string columns in a ``DataFrame`` description.++        >>> df.describe(include=[np.object])+               object+        count       3+        unique      3+        top         c+        freq        1++        Including only categorical columns from a ``DataFrame`` description.++        >>> df.describe(include=['category'])+               categorical+        count            3+        unique           3+        top              f+        freq             1++        Excluding numeric columns from a ``DataFrame`` description.++        >>> df.describe(exclude=[np.number])+               categorical object+        count            3      3+        unique           3      3+        top              f      c+        freq             1      1++        Excluding object columns from a ``DataFrame`` description.++        >>> df.describe(exclude=[np.object])+               categorical  numeric+        count            3      3.0+        unique           3      NaN+        top              f      NaN+        freq             1      NaN+        mean           NaN      2.0+        std            NaN      1.0+        min            NaN      1.0+        25%            NaN      1.5+        50%            NaN      2.0+        75%            NaN      2.5+        max            NaN      3.0+        """+        if self.ndim >= 3:+            msg = "describe is not implemented on Panel objects."+            raise NotImplementedError(msg)+        elif self.ndim == 2 and self.columns.size == 0:+            raise ValueError("Cannot describe a DataFrame without columns")++        if percentiles is not None:+            # explicit conversion of `percentiles` to list+            percentiles = list(percentiles)++            # get them all to be in [0, 1]+            self._check_percentile(percentiles)++            # median should always be included+            if 0.5 not in percentiles:+                percentiles.append(0.5)+            percentiles = np.asarray(percentiles)+        else:+            percentiles = np.array([0.25, 0.5, 0.75])++        # sort and check for duplicates+        unique_pcts = np.unique(percentiles)+        if len(unique_pcts) < len(percentiles):+            raise ValueError("percentiles cannot contain duplicates")+        percentiles = unique_pcts++        formatted_percentiles = format_percentiles(percentiles)++        def describe_numeric_1d(series):+            stat_index = (['count', 'mean', 'std', 'min'] ++                          formatted_percentiles + ['max'])+            d = ([series.count(), series.mean(), series.std(), series.min()] ++                 series.quantile(percentiles).tolist() + [series.max()])+            return pd.Series(d, index=stat_index, name=series.name)++        def describe_categorical_1d(data):+            names = ['count', 'unique']+            objcounts = data.value_counts()+            count_unique = len(objcounts[objcounts != 0])+            result = [data.count(), count_unique]+            if result[1] > 0:+                top, freq = objcounts.index[0], objcounts.iloc[0]++                if is_datetime64_any_dtype(data):+                    tz = data.dt.tz+                    asint = data.dropna().values.view('i8')+                    names += ['top', 'freq', 'first', 'last']+                    result += [tslib.Timestamp(top, tz=tz), freq,+                               tslib.Timestamp(asint.min(), tz=tz),+                               tslib.Timestamp(asint.max(), tz=tz)]+                else:+                    names += ['top', 'freq']+                    result += [top, freq]++            return pd.Series(result, index=names, name=data.name)++        def describe_1d(data):+            if is_bool_dtype(data):+                return describe_categorical_1d(data)+            elif is_numeric_dtype(data):+                return describe_numeric_1d(data)+            elif is_timedelta64_dtype(data):+                return describe_numeric_1d(data)+            else:+                return describe_categorical_1d(data)++        if self.ndim == 1:+            return describe_1d(self)+        elif (include is None) and (exclude is None):+            # when some numerics are found, keep only numerics+            data = self.select_dtypes(include=[np.number])+            if len(data.columns) == 0:+                data = self+        elif include == 'all':+            if exclude is not None:+                msg = "exclude must be None when include is 'all'"+                raise ValueError(msg)+            data = self+        else:+            data = self.select_dtypes(include=include, exclude=exclude)++        ldesc = [describe_1d(s) for _, s in data.iteritems()]+        # set a convenient order for rows+        names = []+        ldesc_indexes = sorted((x.index for x in ldesc), key=len)+        for idxnames in ldesc_indexes:+            for name in idxnames:+                if name not in names:+                    names.append(name)++        d = pd.concat(ldesc, join_axes=pd.Index([names]), axis=1)+        d.columns = data.columns.copy()+        return d++    def _check_percentile(self, q):+        """Validate percentiles (used by describe and quantile)."""++        msg = ("percentiles should all be in the interval [0, 1]. "+               "Try {0} instead.")+        q = np.asarray(q)+        if q.ndim == 0:+            if not 0 <= q <= 1:+                raise ValueError(msg.format(q / 100.0))+        else:+            if not all(0 <= qs <= 1 for qs in q):+                raise ValueError(msg.format(q / 100.0))+        return q++    _shared_docs['pct_change'] = """+        Percentage change between the current and a prior element.++        Computes the percentage change from the immediately previous row by+        default. This is useful in comparing the percentage of change in a time+        series of elements.++        Parameters+        ----------+        periods : int, default 1+            Periods to shift for forming percent change.+        fill_method : str, default 'pad'+            How to handle NAs before computing percent changes.+        limit : int, default None+            The number of consecutive NAs to fill before stopping.+        freq : DateOffset, timedelta, or offset alias string, optional+            Increment to use from time series API (e.g. 'M' or BDay()).+        **kwargs+            Additional keyword arguments are passed into+            `DataFrame.shift` or `Series.shift`.++        Returns+        -------+        chg : Series or DataFrame+            The same type as the calling object.++        See Also+        --------+        Series.diff : Compute the difference of two elements in a Series.+        DataFrame.diff : Compute the difference of two elements in a DataFrame.+        Series.shift : Shift the index by some number of periods.+        DataFrame.shift : Shift the index by some number of periods.++        Examples+        --------+        **Series**++        >>> s = pd.Series([90, 91, 85])+        >>> s+        0    90+        1    91+        2    85+        dtype: int64++        >>> s.pct_change()+        0         NaN+        1    0.011111+        2   -0.065934+        dtype: float64++        >>> s.pct_change(periods=2)+        0         NaN+        1         NaN+        2   -0.055556+        dtype: float64++        See the percentage change in a Series where filling NAs with last+        valid observation forward to next valid.++        >>> s = pd.Series([90, 91, None, 85])+        >>> s+        0    90.0+        1    91.0+        2     NaN+        3    85.0+        dtype: float64++        >>> s.pct_change(fill_method='ffill')+        0         NaN+        1    0.011111+        2    0.000000+        3   -0.065934+        dtype: float64++        **DataFrame**++        Percentage change in French franc, Deutsche Mark, and Italian lira from+        1980-01-01 to 1980-03-01.++        >>> df = pd.DataFrame({+        ...     'FR': [4.0405, 4.0963, 4.3149],+        ...     'GR': [1.7246, 1.7482, 1.8519],+        ...     'IT': [804.74, 810.01, 860.13]},+        ...     index=['1980-01-01', '1980-02-01', '1980-03-01'])+        >>> df+                        FR      GR      IT+        1980-01-01  4.0405  1.7246  804.74+        1980-02-01  4.0963  1.7482  810.01+        1980-03-01  4.3149  1.8519  860.13++        >>> df.pct_change()+                          FR        GR        IT+        1980-01-01       NaN       NaN       NaN+        1980-02-01  0.013810  0.013684  0.006549+        1980-03-01  0.053365  0.059318  0.061876++        Percentage of change in GOOG and APPL stock volume. Shows computing+        the percentage change between columns.++        >>> df = pd.DataFrame({+        ...     '2016': [1769950, 30586265],+        ...     '2015': [1500923, 40912316],+        ...     '2014': [1371819, 41403351]},+        ...     index=['GOOG', 'APPL'])+        >>> df+                  2016      2015      2014+        GOOG   1769950   1500923   1371819+        APPL  30586265  40912316  41403351++        >>> df.pct_change(axis='columns')+              2016      2015      2014+        GOOG   NaN -0.151997 -0.086016+        APPL   NaN  0.337604  0.012002+        """++    @Appender(_shared_docs['pct_change'] % _shared_doc_kwargs)+    def pct_change(self, periods=1, fill_method='pad', limit=None, freq=None,+                   **kwargs):+        # TODO: Not sure if above is correct - need someone to confirm.+        axis = self._get_axis_number(kwargs.pop('axis', self._stat_axis_name))+        if fill_method is None:+            data = self+        else:+            data = self.fillna(method=fill_method, limit=limit, axis=axis)++        rs = (data.div(data.shift(periods=periods, freq=freq, axis=axis,+                                  **kwargs)) - 1)+        rs = rs.reindex_like(data)+        if freq is None:+            mask = isna(com.values_from_object(data))+            np.putmask(rs.values, mask, np.nan)+        return rs++    def _agg_by_level(self, name, axis=0, level=0, skipna=True, **kwargs):+        if axis is None:+            raise ValueError("Must specify 'axis' when aggregating by level.")+        grouped = self.groupby(level=level, axis=axis, sort=False)+        if hasattr(grouped, name) and skipna:+            return getattr(grouped, name)(**kwargs)+        axis = self._get_axis_number(axis)+        method = getattr(type(self), name)+        applyf = lambda x: method(x, axis=axis, skipna=skipna, **kwargs)+        return grouped.aggregate(applyf)++    @classmethod+    def _add_numeric_operations(cls):+        """Add the operations to the cls; evaluate the doc strings again"""++        axis_descr, name, name2 = _doc_parms(cls)++        cls.any = _make_logical_function(+            cls, 'any', name, name2, axis_descr,+            _any_desc, nanops.nanany, _any_examples, _any_see_also)+        cls.all = _make_logical_function(+            cls, 'all', name, name2, axis_descr, _all_doc,+            nanops.nanall, _all_examples, _all_see_also)++        @Substitution(outname='mad',+                      desc="Return the mean absolute deviation of the values "+                           "for the requested axis",+                      name1=name, name2=name2, axis_descr=axis_descr,+                      min_count='', examples='')+        @Appender(_num_doc)+        def mad(self, axis=None, skipna=None, level=None):+            if skipna is None:+                skipna = True+            if axis is None:+                axis = self._stat_axis_number+            if level is not None:+                return self._agg_by_level('mad', axis=axis, level=level,+                                          skipna=skipna)++            data = self._get_numeric_data()+            if axis == 0:+                demeaned = data - data.mean(axis=0)+            else:+                demeaned = data.sub(data.mean(axis=1), axis=0)+            return np.abs(demeaned).mean(axis=axis, skipna=skipna)++        cls.mad = mad++        cls.sem = _make_stat_function_ddof(+            cls, 'sem', name, name2, axis_descr,+            "Return unbiased standard error of the mean over requested "+            "axis.\n\nNormalized by N-1 by default. This can be changed "+            "using the ddof argument",+            nanops.nansem)+        cls.var = _make_stat_function_ddof(+            cls, 'var', name, name2, axis_descr,+            "Return unbiased variance over requested axis.\n\nNormalized by "+            "N-1 by default. This can be changed using the ddof argument",+            nanops.nanvar)+        cls.std = _make_stat_function_ddof(+            cls, 'std', name, name2, axis_descr,+            "Return sample standard deviation over requested axis."+            "\n\nNormalized by N-1 by default. This can be changed using the "+            "ddof argument",+            nanops.nanstd)++        @Substitution(outname='compounded',+                      desc="Return the compound percentage of the values for "+                      "the requested axis", name1=name, name2=name2,+                      axis_descr=axis_descr,+                      min_count='', examples='')+        @Appender(_num_doc)+        def compound(self, axis=None, skipna=None, level=None):+            if skipna is None:+                skipna = True+            return (1 + self).prod(axis=axis, skipna=skipna, level=level) - 1++        cls.compound = compound++        cls.cummin = _make_cum_function(+            cls, 'cummin', name, name2, axis_descr, "minimum",+            lambda y, axis: np.minimum.accumulate(y, axis), "min",+            np.inf, np.nan, _cummin_examples)+        cls.cumsum = _make_cum_function(+            cls, 'cumsum', name, name2, axis_descr, "sum",+            lambda y, axis: y.cumsum(axis), "sum", 0.,+            np.nan, _cumsum_examples)+        cls.cumprod = _make_cum_function(+            cls, 'cumprod', name, name2, axis_descr, "product",+            lambda y, axis: y.cumprod(axis), "prod", 1.,+            np.nan, _cumprod_examples)+        cls.cummax = _make_cum_function(+            cls, 'cummax', name, name2, axis_descr, "maximum",+            lambda y, axis: np.maximum.accumulate(y, axis), "max",+            -np.inf, np.nan, _cummax_examples)++        cls.sum = _make_min_count_stat_function(+            cls, 'sum', name, name2, axis_descr,+            'Return the sum of the values for the requested axis',+            nanops.nansum, _sum_examples)+        cls.mean = _make_stat_function(+            cls, 'mean', name, name2, axis_descr,+            'Return the mean of the values for the requested axis',+            nanops.nanmean)+        cls.skew = _make_stat_function(+            cls, 'skew', name, name2, axis_descr,+            'Return unbiased skew over requested axis\nNormalized by N-1',+            nanops.nanskew)+        cls.kurt = _make_stat_function(+            cls, 'kurt', name, name2, axis_descr,+            "Return unbiased kurtosis over requested axis using Fisher's "+            "definition of\nkurtosis (kurtosis of normal == 0.0). Normalized "+            "by N-1\n",+            nanops.nankurt)+        cls.kurtosis = cls.kurt+        cls.prod = _make_min_count_stat_function(+            cls, 'prod', name, name2, axis_descr,+            'Return the product of the values for the requested axis',+            nanops.nanprod, _prod_examples)+        cls.product = cls.prod+        cls.median = _make_stat_function(+            cls, 'median', name, name2, axis_descr,+            'Return the median of the values for the requested axis',+            nanops.nanmedian)+        cls.max = _make_stat_function(+            cls, 'max', name, name2, axis_descr,+            """This method returns the maximum of the values in the object.+            If you want the *index* of the maximum, use ``idxmax``. This is+            the equivalent of the ``numpy.ndarray`` method ``argmax``.""",+            nanops.nanmax)+        cls.min = _make_stat_function(+            cls, 'min', name, name2, axis_descr,+            """This method returns the minimum of the values in the object.+            If you want the *index* of the minimum, use ``idxmin``. This is+            the equivalent of the ``numpy.ndarray`` method ``argmin``.""",+            nanops.nanmin)++    @classmethod+    def _add_series_only_operations(cls):+        """Add the series only operations to the cls; evaluate the doc+        strings again.+        """++        axis_descr, name, name2 = _doc_parms(cls)++        def nanptp(values, axis=0, skipna=True):+            nmax = nanops.nanmax(values, axis, skipna)+            nmin = nanops.nanmin(values, axis, skipna)+            warnings.warn("Method .ptp is deprecated and will be removed "+                          "in a future version. Use numpy.ptp instead.",+                          FutureWarning, stacklevel=4)+            return nmax - nmin++        cls.ptp = _make_stat_function(+            cls, 'ptp', name, name2, axis_descr,+            """+            Returns the difference between the maximum value and the+            minimum value in the object. This is the equivalent of the+            ``numpy.ndarray`` method ``ptp``.++            .. deprecated:: 0.24.0+                Use numpy.ptp instead+            """,+            nanptp)++    @classmethod+    def _add_series_or_dataframe_operations(cls):+        """Add the series or dataframe only operations to the cls; evaluate+        the doc strings again.+        """++        from pandas.core import window as rwindow++        @Appender(rwindow.rolling.__doc__)+        def rolling(self, window, min_periods=None, center=False,+                    win_type=None, on=None, axis=0, closed=None):+            axis = self._get_axis_number(axis)+            return rwindow.rolling(self, window=window,+                                   min_periods=min_periods,+                                   center=center, win_type=win_type,+                                   on=on, axis=axis, closed=closed)++        cls.rolling = rolling++        @Appender(rwindow.expanding.__doc__)+        def expanding(self, min_periods=1, center=False, axis=0):+            axis = self._get_axis_number(axis)+            return rwindow.expanding(self, min_periods=min_periods,+                                     center=center, axis=axis)++        cls.expanding = expanding++        @Appender(rwindow.ewm.__doc__)+        def ewm(self, com=None, span=None, halflife=None, alpha=None,+                min_periods=0, adjust=True, ignore_na=False,+                axis=0):+            axis = self._get_axis_number(axis)+            return rwindow.ewm(self, com=com, span=span, halflife=halflife,+                               alpha=alpha, min_periods=min_periods,+                               adjust=adjust, ignore_na=ignore_na, axis=axis)++        cls.ewm = ewm++    @Appender(_shared_docs['transform'] % _shared_doc_kwargs)+    def transform(self, func, *args, **kwargs):+        result = self.agg(func, *args, **kwargs)+        if is_scalar(result) or len(result) != len(self):+            raise ValueError("transforms cannot produce "+                             "aggregated results")++        return result++    # ----------------------------------------------------------------------+    # Misc methods++    _shared_docs['valid_index'] = """+        Return index for %(position)s non-NA/null value.++        Notes+        --------+        If all elements are non-NA/null, returns None.+        Also returns None for empty %(klass)s.++        Returns+        --------+        scalar : type of index+        """++    def _find_valid_index(self, how):+        """Retrieves the index of the first valid value.++        Parameters+        ----------+        how : {'first', 'last'}+            Use this parameter to change between the first or last valid index.++        Returns+        -------+        idx_first_valid : type of index+        """+        assert how in ['first', 'last']++        if len(self) == 0:  # early stop+            return None+        is_valid = ~self.isna()++        if self.ndim == 2:+            is_valid = is_valid.any(1)  # reduce axis 1++        if how == 'first':+            idxpos = is_valid.values[::].argmax()++        if how == 'last':+            idxpos = len(self) - 1 - is_valid.values[::-1].argmax()++        chk_notna = is_valid.iat[idxpos]+        idx = self.index[idxpos]++        if not chk_notna:+            return None+        return idx++    @Appender(_shared_docs['valid_index'] % {'position': 'first',+                                             'klass': 'NDFrame'})+    def first_valid_index(self):+        return self._find_valid_index('first')++    @Appender(_shared_docs['valid_index'] % {'position': 'last',+                                             'klass': 'NDFrame'})+    def last_valid_index(self):+        return self._find_valid_index('last')++    def to_csv(self, path_or_buf=None, sep=",", na_rep='', float_format=None,+               columns=None, header=True, index=True, index_label=None,+               mode='w', encoding=None, compression='infer', quoting=None,+               quotechar='"', line_terminator='\n', chunksize=None,+               tupleize_cols=None, date_format=None, doublequote=True,+               escapechar=None, decimal='.'):+        r"""Write object to a comma-separated values (csv) file++        Parameters+        ----------+        path_or_buf : string or file handle, default None+            File path or object, if None is provided the result is returned as+            a string.+            .. versionchanged:: 0.24.0+                Was previously named "path" for Series.+        sep : character, default ','+            Field delimiter for the output file.+        na_rep : string, default ''+            Missing data representation+        float_format : string, default None+            Format string for floating point numbers+        columns : sequence, optional+            Columns to write+        header : boolean or list of string, default True+            Write out the column names. If a list of strings is given it is+            assumed to be aliases for the column names+            .. versionchanged:: 0.24.0+                Previously defaulted to False for Series.+        index : boolean, default True+            Write row names (index)+        index_label : string or sequence, or False, default None+            Column label for index column(s) if desired. If None is given, and+            `header` and `index` are True, then the index names are used. A+            sequence should be given if the object uses MultiIndex.  If+            False do not print fields for index names. Use index_label=False+            for easier importing in R+        mode : str+            Python write mode, default 'w'+        encoding : string, optional+            A string representing the encoding to use in the output file,+            defaults to 'ascii' on Python 2 and 'utf-8' on Python 3.+        compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None},+                      default 'infer'+            If 'infer' and `path_or_buf` is path-like, then detect compression+            from the following extensions: '.gz', '.bz2', '.zip' or '.xz'+            (otherwise no compression).++            .. versionchanged:: 0.24.0+               'infer' option added and set to default+        line_terminator : string, default ``'\n'``+            The newline character or character sequence to use in the output+            file+        quoting : optional constant from csv module+            defaults to csv.QUOTE_MINIMAL. If you have set a `float_format`+            then floats are converted to strings and thus csv.QUOTE_NONNUMERIC+            will treat them as non-numeric+        quotechar : string (length 1), default '\"'+            character used to quote fields+        doublequote : boolean, default True+            Control quoting of `quotechar` inside a field+        escapechar : string (length 1), default None+            character used to escape `sep` and `quotechar` when appropriate+        chunksize : int or None+            rows to write at a time+        tupleize_cols : boolean, default False+            .. deprecated:: 0.21.0+               This argument will be removed and will always write each row+               of the multi-index as a separate row in the CSV file.++            Write MultiIndex columns as a list of tuples (if True) or in+            the new, expanded format, where each MultiIndex column is a row+            in the CSV (if False).+        date_format : string, default None+            Format string for datetime objects+        decimal: string, default '.'+            Character recognized as decimal separator. E.g. use ',' for+            European data++        .. versionchanged:: 0.24.0+            The order of arguments for Series was changed.+        """++        df = self if isinstance(self, ABCDataFrame) else self.to_frame()++        if tupleize_cols is not None:+            warnings.warn("The 'tupleize_cols' parameter is deprecated and "+                          "will be removed in a future version",+                          FutureWarning, stacklevel=2)+        else:+            tupleize_cols = False++        from pandas.io.formats.csvs import CSVFormatter+        formatter = CSVFormatter(df, path_or_buf,+                                 line_terminator=line_terminator, sep=sep,+                                 encoding=encoding,+                                 compression=compression, quoting=quoting,+                                 na_rep=na_rep, float_format=float_format,+                                 cols=columns, header=header, index=index,+                                 index_label=index_label, mode=mode,+                                 chunksize=chunksize, quotechar=quotechar,+                                 tupleize_cols=tupleize_cols,+                                 date_format=date_format,+                                 doublequote=doublequote,+                                 escapechar=escapechar, decimal=decimal)+        formatter.save()++        if path_or_buf is None:+            return formatter.path_or_buf.getvalue()+++def _doc_parms(cls):+    """Return a tuple of the doc parms."""+    axis_descr = "{%s}" % ', '.join(["{0} ({1})".format(a, i)+                                     for i, a in enumerate(cls._AXIS_ORDERS)])+    name = (cls._constructor_sliced.__name__+            if cls._AXIS_LEN > 1 else 'scalar')+    name2 = cls.__name__+    return axis_descr, name, name2+++_num_doc = """++%(desc)s++Parameters+----------+axis : %(axis_descr)s+skipna : boolean, default True+    Exclude NA/null values when computing the result.+level : int or level name, default None+    If the axis is a MultiIndex (hierarchical), count along a+    particular level, collapsing into a %(name1)s+numeric_only : boolean, default None+    Include only float, int, boolean columns. If None, will attempt to use+    everything, then use only numeric data. Not implemented for Series.+%(min_count)s\++Returns+-------+%(outname)s : %(name1)s or %(name2)s (if level specified)++%(examples)s"""++_num_ddof_doc = """++%(desc)s++Parameters+----------+axis : %(axis_descr)s+skipna : boolean, default True+    Exclude NA/null values. If an entire row/column is NA, the result+    will be NA+level : int or level name, default None+    If the axis is a MultiIndex (hierarchical), count along a+    particular level, collapsing into a %(name1)s+ddof : int, default 1+    Delta Degrees of Freedom. The divisor used in calculations is N - ddof,+    where N represents the number of elements.+numeric_only : boolean, default None+    Include only float, int, boolean columns. If None, will attempt to use+    everything, then use only numeric data. Not implemented for Series.++Returns+-------+%(outname)s : %(name1)s or %(name2)s (if level specified)\n"""++_bool_doc = """+%(desc)s++Parameters+----------+axis : {0 or 'index', 1 or 'columns', None}, default 0+    Indicate which axis or axes should be reduced.++    * 0 / 'index' : reduce the index, return a Series whose index is the+      original column labels.+    * 1 / 'columns' : reduce the columns, return a Series whose index is the+      original index.+    * None : reduce all axes, return a scalar.++skipna : boolean, default True+    Exclude NA/null values. If an entire row/column is NA, the result+    will be NA.+level : int or level name, default None+    If the axis is a MultiIndex (hierarchical), count along a+    particular level, collapsing into a %(name1)s.+bool_only : boolean, default None+    Include only boolean columns. If None, will attempt to use everything,+    then use only boolean data. Not implemented for Series.+**kwargs : any, default None+    Additional keywords have no effect but might be accepted for+    compatibility with NumPy.++Returns+-------+%(outname)s : %(name1)s or %(name2)s (if level specified)++%(see_also)s+%(examples)s"""++_all_doc = """\+Return whether all elements are True, potentially over an axis.++Returns True if all elements within a series or along a Dataframe+axis are non-zero, not-empty or not-False."""++_all_examples = """\+Examples+--------+Series++>>> pd.Series([True, True]).all()+True+>>> pd.Series([True, False]).all()+False++DataFrames++Create a dataframe from a dictionary.++>>> df = pd.DataFrame({'col1': [True, True], 'col2': [True, False]})+>>> df+   col1   col2+0  True   True+1  True  False++Default behaviour checks if column-wise values all return True.++>>> df.all()+col1     True+col2    False+dtype: bool++Specify ``axis='columns'`` to check if row-wise values all return True.++>>> df.all(axis='columns')+0     True+1    False+dtype: bool++Or ``axis=None`` for whether every value is True.++>>> df.all(axis=None)+False+"""++_all_see_also = """\+See also+--------+pandas.Series.all : Return True if all elements are True+pandas.DataFrame.any : Return True if one (or more) elements are True+"""++_cnum_doc = """+Return cumulative %(desc)s over a DataFrame or Series axis.++Returns a DataFrame or Series of the same size containing the cumulative+%(desc)s.++Parameters+----------+axis : {0 or 'index', 1 or 'columns'}, default 0+    The index or the name of the axis. 0 is equivalent to None or 'index'.+skipna : boolean, default True+    Exclude NA/null values. If an entire row/column is NA, the result+    will be NA.+*args, **kwargs :+    Additional keywords have no effect but might be accepted for+    compatibility with NumPy.++Returns+-------+%(outname)s : %(name1)s or %(name2)s\n+%(examples)s+See also+--------+pandas.core.window.Expanding.%(accum_func_name)s : Similar functionality+    but ignores ``NaN`` values.+%(name2)s.%(accum_func_name)s : Return the %(desc)s over+    %(name2)s axis.+%(name2)s.cummax : Return cumulative maximum over %(name2)s axis.+%(name2)s.cummin : Return cumulative minimum over %(name2)s axis.+%(name2)s.cumsum : Return cumulative sum over %(name2)s axis.+%(name2)s.cumprod : Return cumulative product over %(name2)s axis.+"""++_cummin_examples = """\+Examples+--------+**Series**++>>> s = pd.Series([2, np.nan, 5, -1, 0])+>>> s+0    2.0+1    NaN+2    5.0+3   -1.0+4    0.0+dtype: float64++By default, NA values are ignored.++>>> s.cummin()+0    2.0+1    NaN+2    2.0+3   -1.0+4   -1.0+dtype: float64++To include NA values in the operation, use ``skipna=False``++>>> s.cummin(skipna=False)+0    2.0+1    NaN+2    NaN+3    NaN+4    NaN+dtype: float64++**DataFrame**++>>> df = pd.DataFrame([[2.0, 1.0],+...                    [3.0, np.nan],+...                    [1.0, 0.0]],+...                    columns=list('AB'))+>>> df+     A    B+0  2.0  1.0+1  3.0  NaN+2  1.0  0.0++By default, iterates over rows and finds the minimum+in each column. This is equivalent to ``axis=None`` or ``axis='index'``.++>>> df.cummin()+     A    B+0  2.0  1.0+1  2.0  NaN+2  1.0  0.0++To iterate over columns and find the minimum in each row,+use ``axis=1``++>>> df.cummin(axis=1)+     A    B+0  2.0  1.0+1  3.0  NaN+2  1.0  0.0+"""++_cumsum_examples = """\+Examples+--------+**Series**++>>> s = pd.Series([2, np.nan, 5, -1, 0])+>>> s+0    2.0+1    NaN+2    5.0+3   -1.0+4    0.0+dtype: float64++By default, NA values are ignored.++>>> s.cumsum()+0    2.0+1    NaN+2    7.0+3    6.0+4    6.0+dtype: float64++To include NA values in the operation, use ``skipna=False``++>>> s.cumsum(skipna=False)+0    2.0+1    NaN+2    NaN+3    NaN+4    NaN+dtype: float64++**DataFrame**++>>> df = pd.DataFrame([[2.0, 1.0],+...                    [3.0, np.nan],+...                    [1.0, 0.0]],+...                    columns=list('AB'))+>>> df+     A    B+0  2.0  1.0+1  3.0  NaN+2  1.0  0.0++By default, iterates over rows and finds the sum+in each column. This is equivalent to ``axis=None`` or ``axis='index'``.++>>> df.cumsum()+     A    B+0  2.0  1.0+1  5.0  NaN+2  6.0  1.0++To iterate over columns and find the sum in each row,+use ``axis=1``++>>> df.cumsum(axis=1)+     A    B+0  2.0  3.0+1  3.0  NaN+2  1.0  1.0+"""++_cumprod_examples = """\+Examples+--------+**Series**++>>> s = pd.Series([2, np.nan, 5, -1, 0])+>>> s+0    2.0+1    NaN+2    5.0+3   -1.0+4    0.0+dtype: float64++By default, NA values are ignored.++>>> s.cumprod()+0     2.0+1     NaN+2    10.0+3   -10.0+4    -0.0+dtype: float64++To include NA values in the operation, use ``skipna=False``++>>> s.cumprod(skipna=False)+0    2.0+1    NaN+2    NaN+3    NaN+4    NaN+dtype: float64++**DataFrame**++>>> df = pd.DataFrame([[2.0, 1.0],+...                    [3.0, np.nan],+...                    [1.0, 0.0]],+...                    columns=list('AB'))+>>> df+     A    B+0  2.0  1.0+1  3.0  NaN+2  1.0  0.0++By default, iterates over rows and finds the product+in each column. This is equivalent to ``axis=None`` or ``axis='index'``.++>>> df.cumprod()+     A    B+0  2.0  1.0+1  6.0  NaN+2  6.0  0.0++To iterate over columns and find the product in each row,+use ``axis=1``++>>> df.cumprod(axis=1)+     A    B+0  2.0  2.0+1  3.0  NaN+2  1.0  0.0+"""++_cummax_examples = """\+Examples+--------+**Series**++>>> s = pd.Series([2, np.nan, 5, -1, 0])+>>> s+0    2.0+1    NaN+2    5.0+3   -1.0+4    0.0+dtype: float64++By default, NA values are ignored.++>>> s.cummax()+0    2.0+1    NaN+2    5.0+3    5.0+4    5.0+dtype: float64++To include NA values in the operation, use ``skipna=False``++>>> s.cummax(skipna=False)+0    2.0+1    NaN+2    NaN+3    NaN+4    NaN+dtype: float64++**DataFrame**++>>> df = pd.DataFrame([[2.0, 1.0],+...                    [3.0, np.nan],+...                    [1.0, 0.0]],+...                    columns=list('AB'))+>>> df+     A    B+0  2.0  1.0+1  3.0  NaN+2  1.0  0.0++By default, iterates over rows and finds the maximum+in each column. This is equivalent to ``axis=None`` or ``axis='index'``.++>>> df.cummax()+     A    B+0  2.0  1.0+1  3.0  NaN+2  3.0  1.0++To iterate over columns and find the maximum in each row,+use ``axis=1``++>>> df.cummax(axis=1)+     A    B+0  2.0  2.0+1  3.0  NaN+2  1.0  1.0+"""++_any_see_also = """\+See Also+--------+numpy.any : Numpy version of this method.+Series.any : Return whether any element is True.+Series.all : Return whether all elements are True.+DataFrame.any : Return whether any element is True over requested axis.+DataFrame.all : Return whether all elements are True over requested axis.+"""++_any_desc = """\+Return whether any element is True over requested axis.++Unlike :meth:`DataFrame.all`, this performs an *or* operation. If any of the+values along the specified axis is True, this will return True."""++_any_examples = """\+Examples+--------+**Series**++For Series input, the output is a scalar indicating whether any element+is True.++>>> pd.Series([True, False]).any()+True++**DataFrame**++Whether each column contains at least one True element (the default).++>>> df = pd.DataFrame({"A": [1, 2], "B": [0, 2], "C": [0, 0]})+>>> df+   A  B  C+0  1  0  0+1  2  2  0++>>> df.any()+A     True+B     True+C    False+dtype: bool++Aggregating over the columns.++>>> df = pd.DataFrame({"A": [True, False], "B": [1, 2]})+>>> df+       A  B+0   True  1+1  False  2++>>> df.any(axis='columns')+0    True+1    True+dtype: bool++>>> df = pd.DataFrame({"A": [True, False], "B": [1, 0]})+>>> df+       A  B+0   True  1+1  False  0++>>> df.any(axis='columns')+0    True+1    False+dtype: bool++Aggregating over the entire DataFrame with ``axis=None``.++>>> df.any(axis=None)+True++`any` for an empty DataFrame is an empty Series.++>>> pd.DataFrame([]).any()+Series([], dtype: bool)+"""++_sum_examples = """\+Examples+--------+By default, the sum of an empty or all-NA Series is ``0``.++>>> pd.Series([]).sum()  # min_count=0 is the default+0.0++This can be controlled with the ``min_count`` parameter. For example, if+you'd like the sum of an empty series to be NaN, pass ``min_count=1``.++>>> pd.Series([]).sum(min_count=1)+nan++Thanks to the ``skipna`` parameter, ``min_count`` handles all-NA and+empty series identically.++>>> pd.Series([np.nan]).sum()+0.0++>>> pd.Series([np.nan]).sum(min_count=1)+nan+"""++_prod_examples = """\+Examples+--------+By default, the product of an empty or all-NA Series is ``1``++>>> pd.Series([]).prod()+1.0++This can be controlled with the ``min_count`` parameter++>>> pd.Series([]).prod(min_count=1)+nan++Thanks to the ``skipna`` parameter, ``min_count`` handles all-NA and+empty series identically.++>>> pd.Series([np.nan]).prod()+1.0++>>> pd.Series([np.nan]).prod(min_count=1)+nan+"""+++_min_count_stub = """\+min_count : int, default 0+    The required number of valid values to perform the operation. If fewer than+    ``min_count`` non-NA values are present the result will be NA.++    .. versionadded :: 0.22.0++       Added with the default being 0. This means the sum of an all-NA+       or empty Series is 0, and the product of an all-NA or empty+       Series is 1.+"""+++def _make_min_count_stat_function(cls, name, name1, name2, axis_descr, desc,+                                  f, examples):+    @Substitution(outname=name, desc=desc, name1=name1, name2=name2,+                  axis_descr=axis_descr, min_count=_min_count_stub,+                  examples=examples)+    @Appender(_num_doc)+    def stat_func(self, axis=None, skipna=None, level=None, numeric_only=None,+                  min_count=0,+                  **kwargs):+        nv.validate_stat_func(tuple(), kwargs, fname=name)+        if skipna is None:+            skipna = True+        if axis is None:+            axis = self._stat_axis_number+        if level is not None:+            return self._agg_by_level(name, axis=axis, level=level,+                                      skipna=skipna, min_count=min_count)+        return self._reduce(f, name, axis=axis, skipna=skipna,+                            numeric_only=numeric_only, min_count=min_count)++    return set_function_name(stat_func, name, cls)+++def _make_stat_function(cls, name, name1, name2, axis_descr, desc, f):+    @Substitution(outname=name, desc=desc, name1=name1, name2=name2,+                  axis_descr=axis_descr, min_count='', examples='')+    @Appender(_num_doc)+    def stat_func(self, axis=None, skipna=None, level=None, numeric_only=None,+                  **kwargs):+        nv.validate_stat_func(tuple(), kwargs, fname=name)+        if skipna is None:+            skipna = True+        if axis is None:+            axis = self._stat_axis_number+        if level is not None:+            return self._agg_by_level(name, axis=axis, level=level,+                                      skipna=skipna)+        return self._reduce(f, name, axis=axis, skipna=skipna,+                            numeric_only=numeric_only)++    return set_function_name(stat_func, name, cls)+++def _make_stat_function_ddof(cls, name, name1, name2, axis_descr, desc, f):+    @Substitution(outname=name, desc=desc, name1=name1, name2=name2,+                  axis_descr=axis_descr)+    @Appender(_num_ddof_doc)+    def stat_func(self, axis=None, skipna=None, level=None, ddof=1,+                  numeric_only=None, **kwargs):+        nv.validate_stat_ddof_func(tuple(), kwargs, fname=name)+        if skipna is None:+            skipna = True+        if axis is None:+            axis = self._stat_axis_number+        if level is not None:+            return self._agg_by_level(name, axis=axis, level=level,+                                      skipna=skipna, ddof=ddof)+        return self._reduce(f, name, axis=axis, numeric_only=numeric_only,+                            skipna=skipna, ddof=ddof)++    return set_function_name(stat_func, name, cls)+++def _make_cum_function(cls, name, name1, name2, axis_descr, desc,+                       accum_func, accum_func_name, mask_a, mask_b, examples):+    @Substitution(outname=name, desc=desc, name1=name1, name2=name2,+                  axis_descr=axis_descr, accum_func_name=accum_func_name,+                  examples=examples)+    @Appender(_cnum_doc)+    def cum_func(self, axis=None, skipna=True, *args, **kwargs):+        skipna = nv.validate_cum_func_with_skipna(skipna, args, kwargs, name)+        if axis is None:+            axis = self._stat_axis_number+        else:+            axis = self._get_axis_number(axis)++        y = com.values_from_object(self).copy()++        if (skipna and+                issubclass(y.dtype.type, (np.datetime64, np.timedelta64))):+            result = accum_func(y, axis)+            mask = isna(self)+            np.putmask(result, mask, tslib.iNaT)+        elif skipna and not issubclass(y.dtype.type, (np.integer, np.bool_)):+            mask = isna(self)+            np.putmask(y, mask, mask_a)+            result = accum_func(y, axis)+            np.putmask(result, mask, mask_b)+        else:+            result = accum_func(y, axis)++        d = self._construct_axes_dict()+        d['copy'] = False+        return self._constructor(result, **d).__finalize__(self)++    return set_function_name(cum_func, name, cls)+++def _make_logical_function(cls, name, name1, name2, axis_descr, desc, f,+                           examples, see_also):+    @Substitution(outname=name, desc=desc, name1=name1, name2=name2,+                  axis_descr=axis_descr, examples=examples, see_also=see_also)+    @Appender(_bool_doc)+    def logical_func(self, axis=0, bool_only=None, skipna=True, level=None,+                     **kwargs):+        nv.validate_logical_func(tuple(), kwargs, fname=name)+        if level is not None:+            if bool_only is not None:+                raise NotImplementedError("Option bool_only is not "+                                          "implemented with option level.")+            return self._agg_by_level(name, axis=axis, level=level,+                                      skipna=skipna)+        return self._reduce(f, name, axis=axis, skipna=skipna,+                            numeric_only=bool_only, filter_type='bool')++    return set_function_name(logical_func, name, cls)+++# install the indexes+for _name, _indexer in indexing.get_indexers_list():+    NDFrame._create_indexer(_name, _indexer)
+ test/files/pandas2.py view
@@ -0,0 +1,7850 @@+"""+DataFrame+---------+An efficient 2D container for potentially mixed-type time series or other+labeled data series.++Similar to its R counterpart, data.frame, except providing automatic data+alignment and a host of useful data manipulation methods having to do with the+labeling information+"""+from __future__ import division+# pylint: disable=E1101,E1103+# pylint: disable=W0212,W0231,W0703,W0622++import functools+import collections+import itertools+import sys+import warnings+from textwrap import dedent++import numpy as np+import numpy.ma as ma++from pandas.core.accessor import CachedAccessor+from pandas.core.dtypes.cast import (+    maybe_upcast,+    cast_scalar_to_array,+    construct_1d_arraylike_from_scalar,+    infer_dtype_from_scalar,+    maybe_cast_to_datetime,+    maybe_infer_to_datetimelike,+    maybe_convert_platform,+    maybe_downcast_to_dtype,+    invalidate_string_dtypes,+    coerce_to_dtypes,+    maybe_upcast_putmask,+    find_common_type)+from pandas.core.dtypes.common import (+    is_categorical_dtype,+    is_object_dtype,+    is_extension_type,+    is_extension_array_dtype,+    is_datetimetz,+    is_datetime64_any_dtype,+    is_bool_dtype,+    is_integer_dtype,+    is_float_dtype,+    is_integer,+    is_scalar,+    is_dtype_equal,+    needs_i8_conversion,+    _get_dtype_from_object,+    ensure_float64,+    ensure_int64,+    ensure_platform_int,+    is_list_like,+    is_nested_list_like,+    is_iterator,+    is_sequence,+    is_named_tuple)+from pandas.core.dtypes.concat import _get_sliced_frame_result_type+from pandas.core.dtypes.missing import isna, notna+++from pandas.core.generic import NDFrame, _shared_docs+from pandas.core.index import (Index, MultiIndex, ensure_index,+                               ensure_index_from_sequences)+from pandas.core.indexing import (maybe_droplevels, convert_to_index_sliceable,+                                  check_bool_indexer)+from pandas.core.internals import (BlockManager,+                                   create_block_manager_from_arrays,+                                   create_block_manager_from_blocks)+from pandas.core.series import Series+from pandas.core.arrays import Categorical, ExtensionArray+import pandas.core.algorithms as algorithms+from pandas.compat import (range, map, zip, lrange, lmap, lzip, StringIO, u,+                           OrderedDict, raise_with_traceback,+                           string_and_binary_types)+from pandas import compat+from pandas.compat import PY36+from pandas.compat.numpy import function as nv+from pandas.util._decorators import (Appender, Substitution,+                                     rewrite_axis_style_signature,+                                     deprecate_kwarg)+from pandas.util._validators import (validate_bool_kwarg,+                                     validate_axis_style_args)++from pandas.core.indexes.period import PeriodIndex+from pandas.core.indexes.datetimes import DatetimeIndex+from pandas.core.indexes.timedeltas import TimedeltaIndex+import pandas.core.indexes.base as ibase++import pandas.core.common as com+import pandas.core.nanops as nanops+import pandas.core.ops as ops+import pandas.io.formats.console as console+import pandas.io.formats.format as fmt+from pandas.io.formats.printing import pprint_thing+import pandas.plotting._core as gfx++from pandas._libs import lib, algos as libalgos++from pandas.core.config import get_option++# ---------------------------------------------------------------------+# Docstring templates++_shared_doc_kwargs = dict(+    axes='index, columns', klass='DataFrame',+    axes_single_arg="{0 or 'index', 1 or 'columns'}",+    axis="""+    axis : {0 or 'index', 1 or 'columns'}, default 0+        - 0 or 'index': apply function to each column.+        - 1 or 'columns': apply function to each row.""",+    optional_by="""+        by : str or list of str+            Name or list of names to sort by.++            - if `axis` is 0 or `'index'` then `by` may contain index+              levels and/or column labels+            - if `axis` is 1 or `'columns'` then `by` may contain column+              levels and/or index labels++            .. versionchanged:: 0.23.0+               Allow specifying index or column level names.""",+    versionadded_to_excel='',+    optional_labels="""labels : array-like, optional+            New labels / index to conform the axis specified by 'axis' to.""",+    optional_axis="""axis : int or str, optional+            Axis to target. Can be either the axis name ('index', 'columns')+            or number (0, 1).""",+)++_numeric_only_doc = """numeric_only : boolean, default None+    Include only float, int, boolean data. If None, will attempt to use+    everything, then use only numeric data+"""++_merge_doc = """+Merge DataFrame or named Series objects by performing a database-style join+operation by columns or indexes.++If joining columns on columns, the DataFrame indexes *will be+ignored*. Otherwise if joining indexes on indexes or indexes on a column or+columns, the index will be passed on.++Parameters+----------%s+right : DataFrame or named Series+    Object to merge with.+how : {'left', 'right', 'outer', 'inner'}, default 'inner'+    Type of merge to be performed.++    * left: use only keys from left frame, similar to a SQL left outer join;+      preserve key order+    * right: use only keys from right frame, similar to a SQL right outer join;+      preserve key order+    * outer: use union of keys from both frames, similar to a SQL full outer+      join; sort keys lexicographically+    * inner: use intersection of keys from both frames, similar to a SQL inner+      join; preserve the order of the left keys+on : label or list+    Column or index level names to join on. These must be found in both+    DataFrames. If `on` is None and not merging on indexes then this defaults+    to the intersection of the columns in both DataFrames.+left_on : label or list, or array-like+    Column or index level names to join on in the left DataFrame. Can also+    be an array or list of arrays of the length of the left DataFrame.+    These arrays are treated as if they are columns.+right_on : label or list, or array-like+    Column or index level names to join on in the right DataFrame. Can also+    be an array or list of arrays of the length of the right DataFrame.+    These arrays are treated as if they are columns.+left_index : boolean, default False+    Use the index from the left DataFrame as the join key(s). If it is a+    MultiIndex, the number of keys in the other DataFrame (either the index+    or a number of columns) must match the number of levels.+right_index : boolean, default False+    Use the index from the right DataFrame as the join key. Same caveats as+    left_index.+sort : boolean, default False+    Sort the join keys lexicographically in the result DataFrame. If False,+    the order of the join keys depends on the join type (how keyword).+suffixes : 2-length sequence (tuple, list, ...)+    Suffix to apply to overlapping column names in the left and right+    side, respectively.+copy : boolean, default True+    If False, avoid copy if possible.+indicator : boolean or string, default False+    If True, adds a column to output DataFrame called "_merge" with+    information on the source of each row.+    If string, column with information on source of each row will be added to+    output DataFrame, and column will be named value of string.+    Information column is Categorical-type and takes on a value of "left_only"+    for observations whose merge key only appears in 'left' DataFrame,+    "right_only" for observations whose merge key only appears in 'right'+    DataFrame, and "both" if the observation's merge key is found in both.++validate : string, default None+    If specified, checks if merge is of specified type.++    * "one_to_one" or "1:1": check if merge keys are unique in both+      left and right datasets.+    * "one_to_many" or "1:m": check if merge keys are unique in left+      dataset.+    * "many_to_one" or "m:1": check if merge keys are unique in right+      dataset.+    * "many_to_many" or "m:m": allowed, but does not result in checks.++    .. versionadded:: 0.21.0++Returns+-------+DataFrame++Notes+-----+Support for specifying index levels as the `on`, `left_on`, and+`right_on` parameters was added in version 0.23.0+Support for merging named Series objects was added in version 0.24.0++See Also+--------+merge_ordered : merge with optional filling/interpolation.+merge_asof : merge on nearest keys.+DataFrame.join : similar method using indices.++Examples+--------++>>> A = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'],+...                   'value': [1, 2, 3, 5]})+>>> B = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'],+...                   'value': [5, 6, 7, 8]})+>>> A+    lkey value+0   foo      1+1   bar      2+2   baz      3+3   foo      5+>>> B+    rkey value+0   foo      5+1   bar      6+2   baz      7+3   foo      8++>>> A.merge(B, left_on='lkey', right_on='rkey', how='outer')+  lkey  value_x rkey  value_y+0  foo        1  foo        5+1  foo        1  foo        8+2  foo        5  foo        5+3  foo        5  foo        8+4  bar        2  bar        6+5  baz        3  baz        7+"""++# -----------------------------------------------------------------------+# DataFrame class+++class DataFrame(NDFrame):+    """ Two-dimensional size-mutable, potentially heterogeneous tabular data+    structure with labeled axes (rows and columns). Arithmetic operations+    align on both row and column labels. Can be thought of as a dict-like+    container for Series objects. The primary pandas data structure.++    Parameters+    ----------+    data : ndarray (structured or homogeneous), Iterable, dict, or DataFrame+        Dict can contain Series, arrays, constants, or list-like objects++        .. versionchanged :: 0.23.0+           If data is a dict, argument order is maintained for Python 3.6+           and later.++    index : Index or array-like+        Index to use for resulting frame. Will default to RangeIndex if+        no indexing information part of input data and no index provided+    columns : Index or array-like+        Column labels to use for resulting frame. Will default to+        RangeIndex (0, 1, 2, ..., n) if no column labels are provided+    dtype : dtype, default None+        Data type to force. Only a single dtype is allowed. If None, infer+    copy : boolean, default False+        Copy data from inputs. Only affects DataFrame / 2d ndarray input++    Examples+    --------+    Constructing DataFrame from a dictionary.++    >>> d = {'col1': [1, 2], 'col2': [3, 4]}+    >>> df = pd.DataFrame(data=d)+    >>> df+       col1  col2+    0     1     3+    1     2     4++    Notice that the inferred dtype is int64.++    >>> df.dtypes+    col1    int64+    col2    int64+    dtype: object++    To enforce a single dtype:++    >>> df = pd.DataFrame(data=d, dtype=np.int8)+    >>> df.dtypes+    col1    int8+    col2    int8+    dtype: object++    Constructing DataFrame from numpy ndarray:++    >>> df2 = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),+    ...                    columns=['a', 'b', 'c'])+    >>> df2+       a  b  c+    0  1  2  3+    1  4  5  6+    2  7  8  9++    See also+    --------+    DataFrame.from_records : constructor from tuples, also record arrays+    DataFrame.from_dict : from dicts of Series, arrays, or dicts+    DataFrame.from_items : from sequence of (key, value) pairs+    pandas.read_csv, pandas.read_table, pandas.read_clipboard+    """++    @property+    def _constructor(self):+        return DataFrame++    _constructor_sliced = Series+    _deprecations = NDFrame._deprecations | frozenset(+        ['sortlevel', 'get_value', 'set_value', 'from_csv', 'from_items'])+    _accessors = set()++    @property+    def _constructor_expanddim(self):+        from pandas.core.panel import Panel+        return Panel++    def __init__(self, data=None, index=None, columns=None, dtype=None,+                 copy=False):+        if data is None:+            data = {}+        if dtype is not None:+            dtype = self._validate_dtype(dtype)++        if isinstance(data, DataFrame):+            data = data._data++        if isinstance(data, BlockManager):+            mgr = self._init_mgr(data, axes=dict(index=index, columns=columns),+                                 dtype=dtype, copy=copy)+        elif isinstance(data, dict):+            mgr = self._init_dict(data, index, columns, dtype=dtype)+        elif isinstance(data, ma.MaskedArray):+            import numpy.ma.mrecords as mrecords+            # masked recarray+            if isinstance(data, mrecords.MaskedRecords):+                mgr = _masked_rec_array_to_mgr(data, index, columns, dtype,+                                               copy)++            # a masked array+            else:+                mask = ma.getmaskarray(data)+                if mask.any():+                    data, fill_value = maybe_upcast(data, copy=True)+                    data[mask] = fill_value+                else:+                    data = data.copy()+                mgr = self._init_ndarray(data, index, columns, dtype=dtype,+                                         copy=copy)++        elif isinstance(data, (np.ndarray, Series, Index)):+            if data.dtype.names:+                data_columns = list(data.dtype.names)+                data = {k: data[k] for k in data_columns}+                if columns is None:+                    columns = data_columns+                mgr = self._init_dict(data, index, columns, dtype=dtype)+            elif getattr(data, 'name', None) is not None:+                mgr = self._init_dict({data.name: data}, index, columns,+                                      dtype=dtype)+            else:+                mgr = self._init_ndarray(data, index, columns, dtype=dtype,+                                         copy=copy)++        # For data is list-like, or Iterable (will consume into list)+        elif (isinstance(data, collections.Iterable)+              and not isinstance(data, string_and_binary_types)):+            if not isinstance(data, collections.Sequence):+                data = list(data)+            if len(data) > 0:+                if is_list_like(data[0]) and getattr(data[0], 'ndim', 1) == 1:+                    if is_named_tuple(data[0]) and columns is None:+                        columns = data[0]._fields+                    arrays, columns = _to_arrays(data, columns, dtype=dtype)+                    columns = ensure_index(columns)++                    # set the index+                    if index is None:+                        if isinstance(data[0], Series):+                            index = _get_names_from_index(data)+                        elif isinstance(data[0], Categorical):+                            index = ibase.default_index(len(data[0]))+                        else:+                            index = ibase.default_index(len(data))++                    mgr = _arrays_to_mgr(arrays, columns, index, columns,+                                         dtype=dtype)+                else:+                    mgr = self._init_ndarray(data, index, columns, dtype=dtype,+                                             copy=copy)+            else:+                mgr = self._init_dict({}, index, columns, dtype=dtype)+        else:+            try:+                arr = np.array(data, dtype=dtype, copy=copy)+            except (ValueError, TypeError) as e:+                exc = TypeError('DataFrame constructor called with '+                                'incompatible data and dtype: {e}'.format(e=e))+                raise_with_traceback(exc)++            if arr.ndim == 0 and index is not None and columns is not None:+                values = cast_scalar_to_array((len(index), len(columns)),+                                              data, dtype=dtype)+                mgr = self._init_ndarray(values, index, columns,+                                         dtype=values.dtype, copy=False)+            else:+                raise ValueError('DataFrame constructor not properly called!')++        NDFrame.__init__(self, mgr, fastpath=True)++    def _init_dict(self, data, index, columns, dtype=None):+        """+        Segregate Series based on type and coerce into matrices.+        Needs to handle a lot of exceptional cases.+        """+        if columns is not None:+            arrays = Series(data, index=columns, dtype=object)+            data_names = arrays.index++            missing = arrays.isnull()+            if index is None:+                # GH10856+                # raise ValueError if only scalars in dict+                index = extract_index(arrays[~missing])+            else:+                index = ensure_index(index)++            # no obvious "empty" int column+            if missing.any() and not is_integer_dtype(dtype):+                if dtype is None or np.issubdtype(dtype, np.flexible):+                    # 1783+                    nan_dtype = object+                else:+                    nan_dtype = dtype+                v = construct_1d_arraylike_from_scalar(np.nan, len(index),+                                                       nan_dtype)+                arrays.loc[missing] = [v] * missing.sum()++        else:+            keys = com.dict_keys_to_ordered_list(data)+            columns = data_names = Index(keys)+            arrays = [data[k] for k in keys]++        return _arrays_to_mgr(arrays, data_names, index, columns, dtype=dtype)++    def _init_ndarray(self, values, index, columns, dtype=None, copy=False):+        # input must be a ndarray, list, Series, index++        if isinstance(values, Series):+            if columns is None:+                if values.name is not None:+                    columns = [values.name]+            if index is None:+                index = values.index+            else:+                values = values.reindex(index)++            # zero len case (GH #2234)+            if not len(values) and columns is not None and len(columns):+                values = np.empty((0, 1), dtype=object)++        # helper to create the axes as indexes+        def _get_axes(N, K, index=index, columns=columns):+            # return axes or defaults++            if index is None:+                index = ibase.default_index(N)+            else:+                index = ensure_index(index)++            if columns is None:+                columns = ibase.default_index(K)+            else:+                columns = ensure_index(columns)+            return index, columns++        # we could have a categorical type passed or coerced to 'category'+        # recast this to an _arrays_to_mgr+        if (is_categorical_dtype(getattr(values, 'dtype', None)) or+                is_categorical_dtype(dtype)):++            if not hasattr(values, 'dtype'):+                values = _prep_ndarray(values, copy=copy)+                values = values.ravel()+            elif copy:+                values = values.copy()++            index, columns = _get_axes(len(values), 1)+            return _arrays_to_mgr([values], columns, index, columns,+                                  dtype=dtype)+        elif (is_datetimetz(values) or is_extension_array_dtype(values)):+            # GH19157+            if columns is None:+                columns = [0]+            return _arrays_to_mgr([values], columns, index, columns,+                                  dtype=dtype)++        # by definition an array here+        # the dtypes will be coerced to a single dtype+        values = _prep_ndarray(values, copy=copy)++        if dtype is not None:+            if not is_dtype_equal(values.dtype, dtype):+                try:+                    values = values.astype(dtype)+                except Exception as orig:+                    e = ValueError("failed to cast to '{dtype}' (Exception "+                                   "was: {orig})".format(dtype=dtype,+                                                         orig=orig))+                    raise_with_traceback(e)++        index, columns = _get_axes(*values.shape)+        values = values.T++        # if we don't have a dtype specified, then try to convert objects+        # on the entire block; this is to convert if we have datetimelike's+        # embedded in an object type+        if dtype is None and is_object_dtype(values):+            values = maybe_infer_to_datetimelike(values)++        return create_block_manager_from_blocks([values], [columns, index])++    @property+    def axes(self):+        """+        Return a list representing the axes of the DataFrame.++        It has the row axis labels and column axis labels as the only members.+        They are returned in that order.++        Examples+        --------+        >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})+        >>> df.axes+        [RangeIndex(start=0, stop=2, step=1), Index(['coll', 'col2'],+        dtype='object')]+        """+        return [self.index, self.columns]++    @property+    def shape(self):+        """+        Return a tuple representing the dimensionality of the DataFrame.++        See Also+        --------+        ndarray.shape++        Examples+        --------+        >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})+        >>> df.shape+        (2, 2)++        >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4],+        ...                    'col3': [5, 6]})+        >>> df.shape+        (2, 3)+        """+        return len(self.index), len(self.columns)++    def _repr_fits_vertical_(self):+        """+        Check length against max_rows.+        """+        max_rows = get_option("display.max_rows")+        return len(self) <= max_rows++    def _repr_fits_horizontal_(self, ignore_width=False):+        """+        Check if full repr fits in horizontal boundaries imposed by the display+        options width and max_columns. In case off non-interactive session, no+        boundaries apply.++        ignore_width is here so ipnb+HTML output can behave the way+        users expect. display.max_columns remains in effect.+        GH3541, GH3573+        """++        width, height = console.get_console_size()+        max_columns = get_option("display.max_columns")+        nb_columns = len(self.columns)++        # exceed max columns+        if ((max_columns and nb_columns > max_columns) or+                ((not ignore_width) and width and nb_columns > (width // 2))):+            return False++        # used by repr_html under IPython notebook or scripts ignore terminal+        # dims+        if ignore_width or not console.in_interactive_session():+            return True++        if (get_option('display.width') is not None or+                console.in_ipython_frontend()):+            # check at least the column row for excessive width+            max_rows = 1+        else:+            max_rows = get_option("display.max_rows")++        # when auto-detecting, so width=None and not in ipython front end+        # check whether repr fits horizontal by actually checking+        # the width of the rendered repr+        buf = StringIO()++        # only care about the stuff we'll actually print out+        # and to_string on entire frame may be expensive+        d = self++        if not (max_rows is None):  # unlimited rows+            # min of two, where one may be None+            d = d.iloc[:min(max_rows, len(d))]+        else:+            return True++        d.to_string(buf=buf)+        value = buf.getvalue()+        repr_width = max(len(l) for l in value.split('\n'))++        return repr_width < width++    def _info_repr(self):+        """True if the repr should show the info view."""+        info_repr_option = (get_option("display.large_repr") == "info")+        return info_repr_option and not (self._repr_fits_horizontal_() and+                                         self._repr_fits_vertical_())++    def __unicode__(self):+        """+        Return a string representation for a particular DataFrame++        Invoked by unicode(df) in py2 only. Yields a Unicode String in both+        py2/py3.+        """+        buf = StringIO(u(""))+        if self._info_repr():+            self.info(buf=buf)+            return buf.getvalue()++        max_rows = get_option("display.max_rows")+        max_cols = get_option("display.max_columns")+        show_dimensions = get_option("display.show_dimensions")+        if get_option("display.expand_frame_repr"):+            width, _ = console.get_console_size()+        else:+            width = None+        self.to_string(buf=buf, max_rows=max_rows, max_cols=max_cols,+                       line_width=width, show_dimensions=show_dimensions)++        return buf.getvalue()++    def _repr_html_(self):+        """+        Return a html representation for a particular DataFrame.+        Mainly for IPython notebook.+        """+        # qtconsole doesn't report its line width, and also+        # behaves badly when outputting an HTML table+        # that doesn't fit the window, so disable it.+        # XXX: In IPython 3.x and above, the Qt console will not attempt to+        # display HTML, so this check can be removed when support for+        # IPython 2.x is no longer needed.+        if console.in_qtconsole():+            # 'HTML output is disabled in QtConsole'+            return None++        if self._info_repr():+            buf = StringIO(u(""))+            self.info(buf=buf)+            # need to escape the <class>, should be the first line.+            val = buf.getvalue().replace('<', r'&lt;', 1)+            val = val.replace('>', r'&gt;', 1)+            return '<pre>' + val + '</pre>'++        if get_option("display.notebook_repr_html"):+            max_rows = get_option("display.max_rows")+            max_cols = get_option("display.max_columns")+            show_dimensions = get_option("display.show_dimensions")++            return self.to_html(max_rows=max_rows, max_cols=max_cols,+                                show_dimensions=show_dimensions, notebook=True)+        else:+            return None++    @property+    def style(self):+        """+        Property returning a Styler object containing methods for+        building a styled HTML representation fo the DataFrame.++        See Also+        --------+        pandas.io.formats.style.Styler+        """+        from pandas.io.formats.style import Styler+        return Styler(self)++    def iteritems(self):+        """+        Iterator over (column name, Series) pairs.++        See also+        --------+        iterrows : Iterate over DataFrame rows as (index, Series) pairs.+        itertuples : Iterate over DataFrame rows as namedtuples of the values.++        """+        if self.columns.is_unique and hasattr(self, '_item_cache'):+            for k in self.columns:+                yield k, self._get_item_cache(k)+        else:+            for i, k in enumerate(self.columns):+                yield k, self._ixs(i, axis=1)++    def iterrows(self):+        """+        Iterate over DataFrame rows as (index, Series) pairs.++        Notes+        -----++        1. Because ``iterrows`` returns a Series for each row,+           it does **not** preserve dtypes across the rows (dtypes are+           preserved across columns for DataFrames). For example,++           >>> df = pd.DataFrame([[1, 1.5]], columns=['int', 'float'])+           >>> row = next(df.iterrows())[1]+           >>> row+           int      1.0+           float    1.5+           Name: 0, dtype: float64+           >>> print(row['int'].dtype)+           float64+           >>> print(df['int'].dtype)+           int64++           To preserve dtypes while iterating over the rows, it is better+           to use :meth:`itertuples` which returns namedtuples of the values+           and which is generally faster than ``iterrows``.++        2. You should **never modify** something you are iterating over.+           This is not guaranteed to work in all cases. Depending on the+           data types, the iterator returns a copy and not a view, and writing+           to it will have no effect.++        Returns+        -------+        it : generator+            A generator that iterates over the rows of the frame.++        See also+        --------+        itertuples : Iterate over DataFrame rows as namedtuples of the values.+        iteritems : Iterate over (column name, Series) pairs.++        """+        columns = self.columns+        klass = self._constructor_sliced+        for k, v in zip(self.index, self.values):+            s = klass(v, index=columns, name=k)+            yield k, s++    def itertuples(self, index=True, name="Pandas"):+        """+        Iterate over DataFrame rows as namedtuples, with index value as first+        element of the tuple.++        Parameters+        ----------+        index : boolean, default True+            If True, return the index as the first element of the tuple.+        name : string, default "Pandas"+            The name of the returned namedtuples or None to return regular+            tuples.++        Notes+        -----+        The column names will be renamed to positional names if they are+        invalid Python identifiers, repeated, or start with an underscore.+        With a large number of columns (>255), regular tuples are returned.++        See also+        --------+        iterrows : Iterate over DataFrame rows as (index, Series) pairs.+        iteritems : Iterate over (column name, Series) pairs.++        Examples+        --------++        >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [0.1, 0.2]},+                              index=['a', 'b'])+        >>> df+           col1  col2+        a     1   0.1+        b     2   0.2+        >>> for row in df.itertuples():+        ...     print(row)+        ...+        Pandas(Index='a', col1=1, col2=0.10000000000000001)+        Pandas(Index='b', col1=2, col2=0.20000000000000001)++        """+        arrays = []+        fields = []+        if index:+            arrays.append(self.index)+            fields.append("Index")++        # use integer indexing because of possible duplicate column names+        arrays.extend(self.iloc[:, k] for k in range(len(self.columns)))++        # Python 3 supports at most 255 arguments to constructor, and+        # things get slow with this many fields in Python 2+        if name is not None and len(self.columns) + index < 256:+            # `rename` is unsupported in Python 2.6+            try:+                itertuple = collections.namedtuple(name,+                                                   fields + list(self.columns),+                                                   rename=True)+                return map(itertuple._make, zip(*arrays))+            except Exception:+                pass++        # fallback to regular tuples+        return zip(*arrays)++    items = iteritems++    def __len__(self):+        """Returns length of info axis, but here we use the index """+        return len(self.index)++    def dot(self, other):+        """+        Matrix multiplication with DataFrame or Series objects.  Can also be+        called using `self @ other` in Python >= 3.5.++        Parameters+        ----------+        other : DataFrame or Series++        Returns+        -------+        dot_product : DataFrame or Series+        """+        if isinstance(other, (Series, DataFrame)):+            common = self.columns.union(other.index)+            if (len(common) > len(self.columns) or+                    len(common) > len(other.index)):+                raise ValueError('matrices are not aligned')++            left = self.reindex(columns=common, copy=False)+            right = other.reindex(index=common, copy=False)+            lvals = left.values+            rvals = right.values+        else:+            left = self+            lvals = self.values+            rvals = np.asarray(other)+            if lvals.shape[1] != rvals.shape[0]:+                raise ValueError('Dot product shape mismatch, '+                                 '{l} vs {r}'.format(l=lvals.shape,+                                                     r=rvals.shape))++        if isinstance(other, DataFrame):+            return self._constructor(np.dot(lvals, rvals), index=left.index,+                                     columns=other.columns)+        elif isinstance(other, Series):+            return Series(np.dot(lvals, rvals), index=left.index)+        elif isinstance(rvals, (np.ndarray, Index)):+            result = np.dot(lvals, rvals)+            if result.ndim == 2:+                return self._constructor(result, index=left.index)+            else:+                return Series(result, index=left.index)+        else:  # pragma: no cover+            raise TypeError('unsupported type: {oth}'.format(oth=type(other)))++    def __matmul__(self, other):+        """ Matrix multiplication using binary `@` operator in Python>=3.5 """+        return self.dot(other)++    def __rmatmul__(self, other):+        """ Matrix multiplication using binary `@` operator in Python>=3.5 """+        return self.T.dot(np.transpose(other)).T++    # ----------------------------------------------------------------------+    # IO methods (to / from other formats)++    @classmethod+    def from_dict(cls, data, orient='columns', dtype=None, columns=None):+        """+        Construct DataFrame from dict of array-like or dicts.++        Creates DataFrame object from dictionary by columns or by index+        allowing dtype specification.++        Parameters+        ----------+        data : dict+            Of the form {field : array-like} or {field : dict}.+        orient : {'columns', 'index'}, default 'columns'+            The "orientation" of the data. If the keys of the passed dict+            should be the columns of the resulting DataFrame, pass 'columns'+            (default). Otherwise if the keys should be rows, pass 'index'.+        dtype : dtype, default None+            Data type to force, otherwise infer.+        columns : list, default None+            Column labels to use when ``orient='index'``. Raises a ValueError+            if used with ``orient='columns'``.++            .. versionadded:: 0.23.0++        Returns+        -------+        pandas.DataFrame++        See Also+        --------+        DataFrame.from_records : DataFrame from ndarray (structured+            dtype), list of tuples, dict, or DataFrame+        DataFrame : DataFrame object creation using constructor++        Examples+        --------+        By default the keys of the dict become the DataFrame columns:++        >>> data = {'col_1': [3, 2, 1, 0], 'col_2': ['a', 'b', 'c', 'd']}+        >>> pd.DataFrame.from_dict(data)+           col_1 col_2+        0      3     a+        1      2     b+        2      1     c+        3      0     d++        Specify ``orient='index'`` to create the DataFrame using dictionary+        keys as rows:++        >>> data = {'row_1': [3, 2, 1, 0], 'row_2': ['a', 'b', 'c', 'd']}+        >>> pd.DataFrame.from_dict(data, orient='index')+               0  1  2  3+        row_1  3  2  1  0+        row_2  a  b  c  d++        When using the 'index' orientation, the column names can be+        specified manually:++        >>> pd.DataFrame.from_dict(data, orient='index',+        ...                        columns=['A', 'B', 'C', 'D'])+               A  B  C  D+        row_1  3  2  1  0+        row_2  a  b  c  d+        """+        index = None+        orient = orient.lower()+        if orient == 'index':+            if len(data) > 0:+                # TODO speed up Series case+                if isinstance(list(data.values())[0], (Series, dict)):+                    data = _from_nested_dict(data)+                else:+                    data, index = list(data.values()), list(data.keys())+        elif orient == 'columns':+            if columns is not None:+                raise ValueError("cannot use columns parameter with "+                                 "orient='columns'")+        else:  # pragma: no cover+            raise ValueError('only recognize index or columns for orient')++        return cls(data, index=index, columns=columns, dtype=dtype)++    def to_dict(self, orient='dict', into=dict):+        """+        Convert the DataFrame to a dictionary.++        The type of the key-value pairs can be customized with the parameters+        (see below).++        Parameters+        ----------+        orient : str {'dict', 'list', 'series', 'split', 'records', 'index'}+            Determines the type of the values of the dictionary.++            - 'dict' (default) : dict like {column -> {index -> value}}+            - 'list' : dict like {column -> [values]}+            - 'series' : dict like {column -> Series(values)}+            - 'split' : dict like+              {'index' -> [index], 'columns' -> [columns], 'data' -> [values]}+            - 'records' : list like+              [{column -> value}, ... , {column -> value}]+            - 'index' : dict like {index -> {column -> value}}++            Abbreviations are allowed. `s` indicates `series` and `sp`+            indicates `split`.++        into : class, default dict+            The collections.Mapping subclass used for all Mappings+            in the return value.  Can be the actual class or an empty+            instance of the mapping type you want.  If you want a+            collections.defaultdict, you must pass it initialized.++            .. versionadded:: 0.21.0++        Returns+        -------+        result : collections.Mapping like {column -> {index -> value}}++        See Also+        --------+        DataFrame.from_dict: create a DataFrame from a dictionary+        DataFrame.to_json: convert a DataFrame to JSON format++        Examples+        --------+        >>> df = pd.DataFrame({'col1': [1, 2],+        ...                    'col2': [0.5, 0.75]},+        ...                   index=['a', 'b'])+        >>> df+           col1  col2+        a     1   0.50+        b     2   0.75+        >>> df.to_dict()+        {'col1': {'a': 1, 'b': 2}, 'col2': {'a': 0.5, 'b': 0.75}}++        You can specify the return orientation.++        >>> df.to_dict('series')+        {'col1': a    1+                 b    2+                 Name: col1, dtype: int64,+         'col2': a    0.50+                 b    0.75+                 Name: col2, dtype: float64}++        >>> df.to_dict('split')+        {'index': ['a', 'b'], 'columns': ['col1', 'col2'],+         'data': [[1.0, 0.5], [2.0, 0.75]]}++        >>> df.to_dict('records')+        [{'col1': 1.0, 'col2': 0.5}, {'col1': 2.0, 'col2': 0.75}]++        >>> df.to_dict('index')+        {'a': {'col1': 1.0, 'col2': 0.5}, 'b': {'col1': 2.0, 'col2': 0.75}}++        You can also specify the mapping type.++        >>> from collections import OrderedDict, defaultdict+        >>> df.to_dict(into=OrderedDict)+        OrderedDict([('col1', OrderedDict([('a', 1), ('b', 2)])),+                     ('col2', OrderedDict([('a', 0.5), ('b', 0.75)]))])++        If you want a `defaultdict`, you need to initialize it:++        >>> dd = defaultdict(list)+        >>> df.to_dict('records', into=dd)+        [defaultdict(<class 'list'>, {'col1': 1.0, 'col2': 0.5}),+         defaultdict(<class 'list'>, {'col1': 2.0, 'col2': 0.75})]+        """+        if not self.columns.is_unique:+            warnings.warn("DataFrame columns are not unique, some "+                          "columns will be omitted.", UserWarning,+                          stacklevel=2)+        # GH16122+        into_c = com.standardize_mapping(into)+        if orient.lower().startswith('d'):+            return into_c(+                (k, v.to_dict(into)) for k, v in compat.iteritems(self))+        elif orient.lower().startswith('l'):+            return into_c((k, v.tolist()) for k, v in compat.iteritems(self))+        elif orient.lower().startswith('sp'):+            return into_c((('index', self.index.tolist()),+                           ('columns', self.columns.tolist()),+                           ('data', lib.map_infer(self.values.ravel(),+                                                  com.maybe_box_datetimelike)+                            .reshape(self.values.shape).tolist())))+        elif orient.lower().startswith('s'):+            return into_c((k, com.maybe_box_datetimelike(v))+                          for k, v in compat.iteritems(self))+        elif orient.lower().startswith('r'):+            return [into_c((k, com.maybe_box_datetimelike(v))+                           for k, v in zip(self.columns, np.atleast_1d(row)))+                    for row in self.values]+        elif orient.lower().startswith('i'):+            return into_c((t[0], dict(zip(self.columns, t[1:])))+                          for t in self.itertuples())+        else:+            raise ValueError("orient '{o}' not understood".format(o=orient))++    def to_gbq(self, destination_table, project_id=None, chunksize=None,+               reauth=False, if_exists='fail', private_key=None,+               auth_local_webserver=False, table_schema=None, location=None,+               progress_bar=True, verbose=None):+        """+        Write a DataFrame to a Google BigQuery table.++        This function requires the `pandas-gbq package+        <https://pandas-gbq.readthedocs.io>`__.++        See the `How to authenticate with Google BigQuery+        <https://pandas-gbq.readthedocs.io/en/latest/howto/authentication.html>`__+        guide for authentication instructions.++        Parameters+        ----------+        destination_table : str+            Name of table to be written, in the form ``dataset.tablename``.+        project_id : str, optional+            Google BigQuery Account project ID. Optional when available from+            the environment.+        chunksize : int, optional+            Number of rows to be inserted in each chunk from the dataframe.+            Set to ``None`` to load the whole dataframe at once.+        reauth : bool, default False+            Force Google BigQuery to re-authenticate the user. This is useful+            if multiple accounts are used.+        if_exists : str, default 'fail'+            Behavior when the destination table exists. Value can be one of:++            ``'fail'``+                If table exists, do nothing.+            ``'replace'``+                If table exists, drop it, recreate it, and insert data.+            ``'append'``+                If table exists, insert data. Create if does not exist.+        private_key : str, optional+            Service account private key in JSON format. Can be file path+            or string contents. This is useful for remote server+            authentication (eg. Jupyter/IPython notebook on remote host).+        auth_local_webserver : bool, default False+            Use the `local webserver flow`_ instead of the `console flow`_+            when getting user credentials.++            .. _local webserver flow:+                http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server+            .. _console flow:+                http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console++            *New in version 0.2.0 of pandas-gbq*.+        table_schema : list of dicts, optional+            List of BigQuery table fields to which according DataFrame+            columns conform to, e.g. ``[{'name': 'col1', 'type':+            'STRING'},...]``. If schema is not provided, it will be+            generated according to dtypes of DataFrame columns. See+            BigQuery API documentation on available names of a field.++            *New in version 0.3.1 of pandas-gbq*.+        location : str, optional+            Location where the load job should run. See the `BigQuery locations+            documentation+            <https://cloud.google.com/bigquery/docs/dataset-locations>`__ for a+            list of available locations. The location must match that of the+            target dataset.++            *New in version 0.5.0 of pandas-gbq*.+        progress_bar : bool, default True+            Use the library `tqdm` to show the progress bar for the upload,+            chunk by chunk.++            *New in version 0.5.0 of pandas-gbq*.+        verbose : bool, deprecated+            Deprecated in Pandas-GBQ 0.4.0. Use the `logging module+            to adjust verbosity instead+            <https://pandas-gbq.readthedocs.io/en/latest/intro.html#logging>`__.++        See Also+        --------+        pandas_gbq.to_gbq : This function in the pandas-gbq library.+        pandas.read_gbq : Read a DataFrame from Google BigQuery.+        """+        from pandas.io import gbq+        return gbq.to_gbq(+            self, destination_table, project_id=project_id,+            chunksize=chunksize, reauth=reauth,+            if_exists=if_exists, private_key=private_key,+            auth_local_webserver=auth_local_webserver,+            table_schema=table_schema, location=location,+            progress_bar=progress_bar, verbose=verbose)++    @classmethod+    def from_records(cls, data, index=None, exclude=None, columns=None,+                     coerce_float=False, nrows=None):+        """+        Convert structured or record ndarray to DataFrame++        Parameters+        ----------+        data : ndarray (structured dtype), list of tuples, dict, or DataFrame+        index : string, list of fields, array-like+            Field of array to use as the index, alternately a specific set of+            input labels to use+        exclude : sequence, default None+            Columns or fields to exclude+        columns : sequence, default None+            Column names to use. If the passed data do not have names+            associated with them, this argument provides names for the+            columns. Otherwise this argument indicates the order of the columns+            in the result (any names not found in the data will become all-NA+            columns)+        coerce_float : boolean, default False+            Attempt to convert values of non-string, non-numeric objects (like+            decimal.Decimal) to floating point, useful for SQL result sets+        nrows : int, default None+            Number of rows to read if data is an iterator++        Returns+        -------+        df : DataFrame+        """++        # Make a copy of the input columns so we can modify it+        if columns is not None:+            columns = ensure_index(columns)++        if is_iterator(data):+            if nrows == 0:+                return cls()++            try:+                first_row = next(data)+            except StopIteration:+                return cls(index=index, columns=columns)++            dtype = None+            if hasattr(first_row, 'dtype') and first_row.dtype.names:+                dtype = first_row.dtype++            values = [first_row]++            if nrows is None:+                values += data+            else:+                values.extend(itertools.islice(data, nrows - 1))++            if dtype is not None:+                data = np.array(values, dtype=dtype)+            else:+                data = values++        if isinstance(data, dict):+            if columns is None:+                columns = arr_columns = ensure_index(sorted(data))+                arrays = [data[k] for k in columns]+            else:+                arrays = []+                arr_columns = []+                for k, v in compat.iteritems(data):+                    if k in columns:+                        arr_columns.append(k)+                        arrays.append(v)++                arrays, arr_columns = _reorder_arrays(arrays, arr_columns,+                                                      columns)++        elif isinstance(data, (np.ndarray, DataFrame)):+            arrays, columns = _to_arrays(data, columns)+            if columns is not None:+                columns = ensure_index(columns)+            arr_columns = columns+        else:+            arrays, arr_columns = _to_arrays(data, columns,+                                             coerce_float=coerce_float)++            arr_columns = ensure_index(arr_columns)+            if columns is not None:+                columns = ensure_index(columns)+            else:+                columns = arr_columns++        if exclude is None:+            exclude = set()+        else:+            exclude = set(exclude)++        result_index = None+        if index is not None:+            if (isinstance(index, compat.string_types) or+                    not hasattr(index, "__iter__")):+                i = columns.get_loc(index)+                exclude.add(index)+                if len(arrays) > 0:+                    result_index = Index(arrays[i], name=index)+                else:+                    result_index = Index([], name=index)+            else:+                try:+                    to_remove = [arr_columns.get_loc(field) for field in index]+                    index_data = [arrays[i] for i in to_remove]+                    result_index = ensure_index_from_sequences(index_data,+                                                               names=index)++                    exclude.update(index)+                except Exception:+                    result_index = index++        if any(exclude):+            arr_exclude = [x for x in exclude if x in arr_columns]+            to_remove = [arr_columns.get_loc(col) for col in arr_exclude]+            arrays = [v for i, v in enumerate(arrays) if i not in to_remove]++            arr_columns = arr_columns.drop(arr_exclude)+            columns = columns.drop(exclude)++        mgr = _arrays_to_mgr(arrays, arr_columns, result_index, columns)++        return cls(mgr)++    def to_records(self, index=True, convert_datetime64=None):+        """+        Convert DataFrame to a NumPy record array.++        Index will be included as the first field of the record array if+        requested.++        Parameters+        ----------+        index : bool, default True+            Include index in resulting record array, stored in 'index'+            field or using the index label, if set.+        convert_datetime64 : bool, default None+            .. deprecated:: 0.23.0++            Whether to convert the index to datetime.datetime if it is a+            DatetimeIndex.++        Returns+        -------+        numpy.recarray+            NumPy ndarray with the DataFrame labels as fields and each row+            of the DataFrame as entries.++        See Also+        --------+        DataFrame.from_records: convert structured or record ndarray+            to DataFrame.+        numpy.recarray: ndarray that allows field access using+            attributes, analogous to typed columns in a+            spreadsheet.++        Examples+        --------+        >>> df = pd.DataFrame({'A': [1, 2], 'B': [0.5, 0.75]},+        ...                   index=['a', 'b'])+        >>> df+           A     B+        a  1  0.50+        b  2  0.75+        >>> df.to_records()+        rec.array([('a', 1, 0.5 ), ('b', 2, 0.75)],+                  dtype=[('index', 'O'), ('A', '<i8'), ('B', '<f8')])++        If the DataFrame index has no label then the recarray field name+        is set to 'index'. If the index has a label then this is used as the+        field name:++        >>> df.index = df.index.rename("I")+        >>> df.to_records()+        rec.array([('a', 1, 0.5 ), ('b', 2, 0.75)],+                  dtype=[('I', 'O'), ('A', '<i8'), ('B', '<f8')])++        The index can be excluded from the record array:++        >>> df.to_records(index=False)+        rec.array([(1, 0.5 ), (2, 0.75)],+                  dtype=[('A', '<i8'), ('B', '<f8')])+        """++        if convert_datetime64 is not None:+            warnings.warn("The 'convert_datetime64' parameter is "+                          "deprecated and will be removed in a future "+                          "version",+                          FutureWarning, stacklevel=2)++        if index:+            if is_datetime64_any_dtype(self.index) and convert_datetime64:+                ix_vals = [self.index.to_pydatetime()]+            else:+                if isinstance(self.index, MultiIndex):+                    # array of tuples to numpy cols. copy copy copy+                    ix_vals = lmap(np.array, zip(*self.index.values))+                else:+                    ix_vals = [self.index.values]++            arrays = ix_vals + [self[c].get_values() for c in self.columns]++            count = 0+            index_names = list(self.index.names)+            if isinstance(self.index, MultiIndex):+                for i, n in enumerate(index_names):+                    if n is None:+                        index_names[i] = 'level_%d' % count+                        count += 1+            elif index_names[0] is None:+                index_names = ['index']+            names = (lmap(compat.text_type, index_names) ++                     lmap(compat.text_type, self.columns))+        else:+            arrays = [self[c].get_values() for c in self.columns]+            names = lmap(compat.text_type, self.columns)++        formats = [v.dtype for v in arrays]+        return np.rec.fromarrays(+            arrays,+            dtype={'names': names, 'formats': formats}+        )++    @classmethod+    def from_items(cls, items, columns=None, orient='columns'):+        """Construct a dataframe from a list of tuples++        .. deprecated:: 0.23.0+          `from_items` is deprecated and will be removed in a future version.+          Use :meth:`DataFrame.from_dict(dict(items)) <DataFrame.from_dict>`+          instead.+          :meth:`DataFrame.from_dict(OrderedDict(items)) <DataFrame.from_dict>`+          may be used to preserve the key order.++        Convert (key, value) pairs to DataFrame. The keys will be the axis+        index (usually the columns, but depends on the specified+        orientation). The values should be arrays or Series.++        Parameters+        ----------+        items : sequence of (key, value) pairs+            Values should be arrays or Series.+        columns : sequence of column labels, optional+            Must be passed if orient='index'.+        orient : {'columns', 'index'}, default 'columns'+            The "orientation" of the data. If the keys of the+            input correspond to column labels, pass 'columns'+            (default). Otherwise if the keys correspond to the index,+            pass 'index'.++        Returns+        -------+        frame : DataFrame+        """++        warnings.warn("from_items is deprecated. Please use "+                      "DataFrame.from_dict(dict(items), ...) instead. "+                      "DataFrame.from_dict(OrderedDict(items)) may be used to "+                      "preserve the key order.",+                      FutureWarning, stacklevel=2)++        keys, values = lzip(*items)++        if orient == 'columns':+            if columns is not None:+                columns = ensure_index(columns)++                idict = dict(items)+                if len(idict) < len(items):+                    if not columns.equals(ensure_index(keys)):+                        raise ValueError('With non-unique item names, passed '+                                         'columns must be identical')+                    arrays = values+                else:+                    arrays = [idict[k] for k in columns if k in idict]+            else:+                columns = ensure_index(keys)+                arrays = values++            # GH 17312+            # Provide more informative error msg when scalar values passed+            try:+                return cls._from_arrays(arrays, columns, None)++            except ValueError:+                if not is_nested_list_like(values):+                    raise ValueError('The value in each (key, value) pair '+                                     'must be an array, Series, or dict')++        elif orient == 'index':+            if columns is None:+                raise TypeError("Must pass columns with orient='index'")++            keys = ensure_index(keys)++            # GH 17312+            # Provide more informative error msg when scalar values passed+            try:+                arr = np.array(values, dtype=object).T+                data = [lib.maybe_convert_objects(v) for v in arr]+                return cls._from_arrays(data, columns, keys)++            except TypeError:+                if not is_nested_list_like(values):+                    raise ValueError('The value in each (key, value) pair '+                                     'must be an array, Series, or dict')++        else:  # pragma: no cover+            raise ValueError("'orient' must be either 'columns' or 'index'")++    @classmethod+    def _from_arrays(cls, arrays, columns, index, dtype=None):+        mgr = _arrays_to_mgr(arrays, columns, index, columns, dtype=dtype)+        return cls(mgr)++    @classmethod+    def from_csv(cls, path, header=0, sep=',', index_col=0, parse_dates=True,+                 encoding=None, tupleize_cols=None,+                 infer_datetime_format=False):+        """Read CSV file.++        .. deprecated:: 0.21.0+            Use :func:`pandas.read_csv` instead.++        It is preferable to use the more powerful :func:`pandas.read_csv`+        for most general purposes, but ``from_csv`` makes for an easy+        roundtrip to and from a file (the exact counterpart of+        ``to_csv``), especially with a DataFrame of time series data.++        This method only differs from the preferred :func:`pandas.read_csv`+        in some defaults:++        - `index_col` is ``0`` instead of ``None`` (take first column as index+          by default)+        - `parse_dates` is ``True`` instead of ``False`` (try parsing the index+          as datetime by default)++        So a ``pd.DataFrame.from_csv(path)`` can be replaced by+        ``pd.read_csv(path, index_col=0, parse_dates=True)``.++        Parameters+        ----------+        path : string file path or file handle / StringIO+        header : int, default 0+            Row to use as header (skip prior rows)+        sep : string, default ','+            Field delimiter+        index_col : int or sequence, default 0+            Column to use for index. If a sequence is given, a MultiIndex+            is used. Different default from read_table+        parse_dates : boolean, default True+            Parse dates. Different default from read_table+        tupleize_cols : boolean, default False+            write multi_index columns as a list of tuples (if True)+            or new (expanded format) if False)+        infer_datetime_format: boolean, default False+            If True and `parse_dates` is True for a column, try to infer the+            datetime format based on the first datetime string. If the format+            can be inferred, there often will be a large parsing speed-up.++        See also+        --------+        pandas.read_csv++        Returns+        -------+        y : DataFrame++        """++        warnings.warn("from_csv is deprecated. Please use read_csv(...) "+                      "instead. Note that some of the default arguments are "+                      "different, so please refer to the documentation "+                      "for from_csv when changing your function calls",+                      FutureWarning, stacklevel=2)++        from pandas.io.parsers import read_csv+        return read_csv(path, header=header, sep=sep,+                        parse_dates=parse_dates, index_col=index_col,+                        encoding=encoding, tupleize_cols=tupleize_cols,+                        infer_datetime_format=infer_datetime_format)++    def to_sparse(self, fill_value=None, kind='block'):+        """+        Convert to SparseDataFrame.++        Implement the sparse version of the DataFrame meaning that any data+        matching a specific value it's omitted in the representation.+        The sparse DataFrame allows for a more efficient storage.++        Parameters+        ----------+        fill_value : float, default None+            The specific value that should be omitted in the representation.+        kind : {'block', 'integer'}, default 'block'+            The kind of the SparseIndex tracking where data is not equal to+            the fill value:++            - 'block' tracks only the locations and sizes of blocks of data.+            - 'integer' keeps an array with all the locations of the data.++            In most cases 'block' is recommended, since it's more memory+            efficient.++        Returns+        -------+        SparseDataFrame+            The sparse representation of the DataFrame.++        See Also+        --------+        DataFrame.to_dense :+            Converts the DataFrame back to the its dense form.++        Examples+        --------+        >>> df = pd.DataFrame([(np.nan, np.nan),+        ...                    (1., np.nan),+        ...                    (np.nan, 1.)])+        >>> df+             0    1+        0  NaN  NaN+        1  1.0  NaN+        2  NaN  1.0+        >>> type(df)+        <class 'pandas.core.frame.DataFrame'>++        >>> sdf = df.to_sparse()+        >>> sdf+             0    1+        0  NaN  NaN+        1  1.0  NaN+        2  NaN  1.0+        >>> type(sdf)+        <class 'pandas.core.sparse.frame.SparseDataFrame'>+        """+        from pandas.core.sparse.frame import SparseDataFrame+        return SparseDataFrame(self._series, index=self.index,+                               columns=self.columns, default_kind=kind,+                               default_fill_value=fill_value)++    def to_panel(self):+        """+        Transform long (stacked) format (DataFrame) into wide (3D, Panel)+        format.++        .. deprecated:: 0.20.0++        Currently the index of the DataFrame must be a 2-level MultiIndex. This+        may be generalized later++        Returns+        -------+        panel : Panel+        """+        # only support this kind for now+        if (not isinstance(self.index, MultiIndex) or  # pragma: no cover+                len(self.index.levels) != 2):+            raise NotImplementedError('Only 2-level MultiIndex are supported.')++        if not self.index.is_unique:+            raise ValueError("Can't convert non-uniquely indexed "+                             "DataFrame to Panel")++        self._consolidate_inplace()++        # minor axis must be sorted+        if self.index.lexsort_depth < 2:+            selfsorted = self.sort_index(level=0)+        else:+            selfsorted = self++        major_axis, minor_axis = selfsorted.index.levels+        major_labels, minor_labels = selfsorted.index.labels+        shape = len(major_axis), len(minor_axis)++        # preserve names, if any+        major_axis = major_axis.copy()+        major_axis.name = self.index.names[0]++        minor_axis = minor_axis.copy()+        minor_axis.name = self.index.names[1]++        # create new axes+        new_axes = [selfsorted.columns, major_axis, minor_axis]++        # create new manager+        new_mgr = selfsorted._data.reshape_nd(axes=new_axes,+                                              labels=[major_labels,+                                                      minor_labels],+                                              shape=shape,+                                              ref_items=selfsorted.columns)++        return self._constructor_expanddim(new_mgr)++    @Appender(_shared_docs['to_excel'] % _shared_doc_kwargs)+    def to_excel(self, excel_writer, sheet_name='Sheet1', na_rep='',+                 float_format=None, columns=None, header=True, index=True,+                 index_label=None, startrow=0, startcol=0, engine=None,+                 merge_cells=True, encoding=None, inf_rep='inf', verbose=True,+                 freeze_panes=None):++        from pandas.io.formats.excel import ExcelFormatter+        formatter = ExcelFormatter(self, na_rep=na_rep, cols=columns,+                                   header=header,+                                   float_format=float_format, index=index,+                                   index_label=index_label,+                                   merge_cells=merge_cells,+                                   inf_rep=inf_rep)+        formatter.write(excel_writer, sheet_name=sheet_name, startrow=startrow,+                        startcol=startcol, freeze_panes=freeze_panes,+                        engine=engine)++    @deprecate_kwarg(old_arg_name='encoding', new_arg_name=None)+    def to_stata(self, fname, convert_dates=None, write_index=True,+                 encoding="latin-1", byteorder=None, time_stamp=None,+                 data_label=None, variable_labels=None, version=114,+                 convert_strl=None):+        """+        Export Stata binary dta files.++        Parameters+        ----------+        fname : path (string), buffer or path object+            string, path object (pathlib.Path or py._path.local.LocalPath) or+            object implementing a binary write() functions. If using a buffer+            then the buffer will not be automatically closed after the file+            data has been written.+        convert_dates : dict+            Dictionary mapping columns containing datetime types to stata+            internal format to use when writing the dates. Options are 'tc',+            'td', 'tm', 'tw', 'th', 'tq', 'ty'. Column can be either an integer+            or a name. Datetime columns that do not have a conversion type+            specified will be converted to 'tc'. Raises NotImplementedError if+            a datetime column has timezone information.+        write_index : bool+            Write the index to Stata dataset.+        encoding : str+            Default is latin-1. Unicode is not supported.+        byteorder : str+            Can be ">", "<", "little", or "big". default is `sys.byteorder`.+        time_stamp : datetime+            A datetime to use as file creation date.  Default is the current+            time.+        data_label : str+            A label for the data set.  Must be 80 characters or smaller.+        variable_labels : dict+            Dictionary containing columns as keys and variable labels as+            values. Each label must be 80 characters or smaller.++            .. versionadded:: 0.19.0++        version : {114, 117}+            Version to use in the output dta file.  Version 114 can be used+            read by Stata 10 and later.  Version 117 can be read by Stata 13+            or later. Version 114 limits string variables to 244 characters or+            fewer while 117 allows strings with lengths up to 2,000,000+            characters.++            .. versionadded:: 0.23.0++        convert_strl : list, optional+            List of column names to convert to string columns to Stata StrL+            format. Only available if version is 117.  Storing strings in the+            StrL format can produce smaller dta files if strings have more than+            8 characters and values are repeated.++            .. versionadded:: 0.23.0++        Raises+        ------+        NotImplementedError+            * If datetimes contain timezone information+            * Column dtype is not representable in Stata+        ValueError+            * Columns listed in convert_dates are neither datetime64[ns]+              or datetime.datetime+            * Column listed in convert_dates is not in DataFrame+            * Categorical label contains more than 32,000 characters++            .. versionadded:: 0.19.0++        See Also+        --------+        pandas.read_stata : Import Stata data files+        pandas.io.stata.StataWriter : low-level writer for Stata data files+        pandas.io.stata.StataWriter117 : low-level writer for version 117 files++        Examples+        --------+        >>> data.to_stata('./data_file.dta')++        Or with dates++        >>> data.to_stata('./date_data_file.dta', {2 : 'tw'})++        Alternatively you can create an instance of the StataWriter class++        >>> writer = StataWriter('./data_file.dta', data)+        >>> writer.write_file()++        With dates:++        >>> writer = StataWriter('./date_data_file.dta', data, {2 : 'tw'})+        >>> writer.write_file()+        """+        kwargs = {}+        if version not in (114, 117):+            raise ValueError('Only formats 114 and 117 supported.')+        if version == 114:+            if convert_strl is not None:+                raise ValueError('strl support is only available when using '+                                 'format 117')+            from pandas.io.stata import StataWriter as statawriter+        else:+            from pandas.io.stata import StataWriter117 as statawriter+            kwargs['convert_strl'] = convert_strl++        writer = statawriter(fname, self, convert_dates=convert_dates,+                             byteorder=byteorder, time_stamp=time_stamp,+                             data_label=data_label, write_index=write_index,+                             variable_labels=variable_labels, **kwargs)+        writer.write_file()++    def to_feather(self, fname):+        """+        write out the binary feather-format for DataFrames++        .. versionadded:: 0.20.0++        Parameters+        ----------+        fname : str+            string file path++        """+        from pandas.io.feather_format import to_feather+        to_feather(self, fname)++    def to_parquet(self, fname, engine='auto', compression='snappy',+                   **kwargs):+        """+        Write a DataFrame to the binary parquet format.++        .. versionadded:: 0.21.0++        This function writes the dataframe as a `parquet file+        <https://parquet.apache.org/>`_. You can choose different parquet+        backends, and have the option of compression. See+        :ref:`the user guide <io.parquet>` for more details.++        Parameters+        ----------+        fname : str+            String file path.+        engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'+            Parquet library to use. If 'auto', then the option+            ``io.parquet.engine`` is used. The default ``io.parquet.engine``+            behavior is to try 'pyarrow', falling back to 'fastparquet' if+            'pyarrow' is unavailable.+        compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy'+            Name of the compression to use. Use ``None`` for no compression.+        **kwargs+            Additional arguments passed to the parquet library. See+            :ref:`pandas io <io.parquet>` for more details.++        See Also+        --------+        read_parquet : Read a parquet file.+        DataFrame.to_csv : Write a csv file.+        DataFrame.to_sql : Write to a sql table.+        DataFrame.to_hdf : Write to hdf.++        Notes+        -----+        This function requires either the `fastparquet+        <https://pypi.org/project/fastparquet>`_ or `pyarrow+        <https://arrow.apache.org/docs/python/>`_ library.++        Examples+        --------+        >>> df = pd.DataFrame(data={'col1': [1, 2], 'col2': [3, 4]})+        >>> df.to_parquet('df.parquet.gzip', compression='gzip')+        >>> pd.read_parquet('df.parquet.gzip')+           col1  col2+        0     1     3+        1     2     4+        """+        from pandas.io.parquet import to_parquet+        to_parquet(self, fname, engine,+                   compression=compression, **kwargs)++    @Substitution(header='Write out the column names. If a list of strings '+                         'is given, it is assumed to be aliases for the '+                         'column names')+    @Substitution(shared_params=fmt.common_docstring,+                  returns=fmt.return_docstring)+    def to_string(self, buf=None, columns=None, col_space=None, header=True,+                  index=True, na_rep='NaN', formatters=None, float_format=None,+                  sparsify=None, index_names=True, justify=None,+                  line_width=None, max_rows=None, max_cols=None,+                  show_dimensions=False):+        """+        Render a DataFrame to a console-friendly tabular output.++        %(shared_params)s+        line_width : int, optional+            Width to wrap a line in characters.++        %(returns)s++        See Also+        --------+        to_html : Convert DataFrame to HTML.++        Examples+        --------+        >>> d = {'col1' : [1, 2, 3], 'col2' : [4, 5, 6]}+        >>> df = pd.DataFrame(d)+        >>> print(df.to_string())+           col1  col2+        0     1     4+        1     2     5+        2     3     6+        """++        formatter = fmt.DataFrameFormatter(self, buf=buf, columns=columns,+                                           col_space=col_space, na_rep=na_rep,+                                           formatters=formatters,+                                           float_format=float_format,+                                           sparsify=sparsify, justify=justify,+                                           index_names=index_names,+                                           header=header, index=index,+                                           line_width=line_width,+                                           max_rows=max_rows,+                                           max_cols=max_cols,+                                           show_dimensions=show_dimensions)+        formatter.to_string()++        if buf is None:+            result = formatter.buf.getvalue()+            return result++    @Substitution(header='whether to print column labels, default True')+    @Substitution(shared_params=fmt.common_docstring,+                  returns=fmt.return_docstring)+    def to_html(self, buf=None, columns=None, col_space=None, header=True,+                index=True, na_rep='NaN', formatters=None, float_format=None,+                sparsify=None, index_names=True, justify=None, bold_rows=True,+                classes=None, escape=True, max_rows=None, max_cols=None,+                show_dimensions=False, notebook=False, decimal='.',+                border=None, table_id=None):+        """+        Render a DataFrame as an HTML table.++        %(shared_params)s+        bold_rows : boolean, default True+            Make the row labels bold in the output+        classes : str or list or tuple, default None+            CSS class(es) to apply to the resulting html table+        escape : boolean, default True+            Convert the characters <, >, and & to HTML-safe sequences.+        notebook : {True, False}, default False+            Whether the generated HTML is for IPython Notebook.+        decimal : string, default '.'+            Character recognized as decimal separator, e.g. ',' in Europe++            .. versionadded:: 0.18.0++        border : int+            A ``border=border`` attribute is included in the opening+            `<table>` tag. Default ``pd.options.html.border``.++            .. versionadded:: 0.19.0++        table_id : str, optional+            A css id is included in the opening `<table>` tag if specified.++            .. versionadded:: 0.23.0++        %(returns)s++        See Also+        --------+        to_string : Convert DataFrame to a string.+        """++        if (justify is not None and+                justify not in fmt._VALID_JUSTIFY_PARAMETERS):+            raise ValueError("Invalid value for justify parameter")++        formatter = fmt.DataFrameFormatter(self, buf=buf, columns=columns,+                                           col_space=col_space, na_rep=na_rep,+                                           formatters=formatters,+                                           float_format=float_format,+                                           sparsify=sparsify, justify=justify,+                                           index_names=index_names,+                                           header=header, index=index,+                                           bold_rows=bold_rows, escape=escape,+                                           max_rows=max_rows,+                                           max_cols=max_cols,+                                           show_dimensions=show_dimensions,+                                           decimal=decimal, table_id=table_id)+        # TODO: a generic formatter wld b in DataFrameFormatter+        formatter.to_html(classes=classes, notebook=notebook, border=border)++        if buf is None:+            return formatter.buf.getvalue()++    def info(self, verbose=None, buf=None, max_cols=None, memory_usage=None,+             null_counts=None):+        """+        Print a concise summary of a DataFrame.++        This method prints information about a DataFrame including+        the index dtype and column dtypes, non-null values and memory usage.++        Parameters+        ----------+        verbose : bool, optional+            Whether to print the full summary. By default, the setting in+            ``pandas.options.display.max_info_columns`` is followed.+        buf : writable buffer, defaults to sys.stdout+            Where to send the output. By default, the output is printed to+            sys.stdout. Pass a writable buffer if you need to further process+            the output.+        max_cols : int, optional+            When to switch from the verbose to the truncated output. If the+            DataFrame has more than `max_cols` columns, the truncated output+            is used. By default, the setting in+            ``pandas.options.display.max_info_columns`` is used.+        memory_usage : bool, str, optional+            Specifies whether total memory usage of the DataFrame+            elements (including the index) should be displayed. By default,+            this follows the ``pandas.options.display.memory_usage`` setting.++            True always show memory usage. False never shows memory usage.+            A value of 'deep' is equivalent to "True with deep introspection".+            Memory usage is shown in human-readable units (base-2+            representation). Without deep introspection a memory estimation is+            made based in column dtype and number of rows assuming values+            consume the same memory amount for corresponding dtypes. With deep+            memory introspection, a real memory usage calculation is performed+            at the cost of computational resources.+        null_counts : bool, optional+            Whether to show the non-null counts. By default, this is shown+            only if the frame is smaller than+            ``pandas.options.display.max_info_rows`` and+            ``pandas.options.display.max_info_columns``. A value of True always+            shows the counts, and False never shows the counts.++        Returns+        -------+        None+            This method prints a summary of a DataFrame and returns None.++        See Also+        --------+        DataFrame.describe: Generate descriptive statistics of DataFrame+            columns.+        DataFrame.memory_usage: Memory usage of DataFrame columns.++        Examples+        --------+        >>> int_values = [1, 2, 3, 4, 5]+        >>> text_values = ['alpha', 'beta', 'gamma', 'delta', 'epsilon']+        >>> float_values = [0.0, 0.25, 0.5, 0.75, 1.0]+        >>> df = pd.DataFrame({"int_col": int_values, "text_col": text_values,+        ...                   "float_col": float_values})+        >>> df+           int_col text_col  float_col+        0        1    alpha       0.00+        1        2     beta       0.25+        2        3    gamma       0.50+        3        4    delta       0.75+        4        5  epsilon       1.00++        Prints information of all columns:++        >>> df.info(verbose=True)+        <class 'pandas.core.frame.DataFrame'>+        RangeIndex: 5 entries, 0 to 4+        Data columns (total 3 columns):+        int_col      5 non-null int64+        text_col     5 non-null object+        float_col    5 non-null float64+        dtypes: float64(1), int64(1), object(1)+        memory usage: 200.0+ bytes++        Prints a summary of columns count and its dtypes but not per column+        information:++        >>> df.info(verbose=False)+        <class 'pandas.core.frame.DataFrame'>+        RangeIndex: 5 entries, 0 to 4+        Columns: 3 entries, int_col to float_col+        dtypes: float64(1), int64(1), object(1)+        memory usage: 200.0+ bytes++        Pipe output of DataFrame.info to buffer instead of sys.stdout, get+        buffer content and writes to a text file:++        >>> import io+        >>> buffer = io.StringIO()+        >>> df.info(buf=buffer)+        >>> s = buffer.getvalue()+        >>> with open("df_info.txt", "w", encoding="utf-8") as f:+        ...     f.write(s)+        260++        The `memory_usage` parameter allows deep introspection mode, specially+        useful for big DataFrames and fine-tune memory optimization:++        >>> random_strings_array = np.random.choice(['a', 'b', 'c'], 10 ** 6)+        >>> df = pd.DataFrame({+        ...     'column_1': np.random.choice(['a', 'b', 'c'], 10 ** 6),+        ...     'column_2': np.random.choice(['a', 'b', 'c'], 10 ** 6),+        ...     'column_3': np.random.choice(['a', 'b', 'c'], 10 ** 6)+        ... })+        >>> df.info()+        <class 'pandas.core.frame.DataFrame'>+        RangeIndex: 1000000 entries, 0 to 999999+        Data columns (total 3 columns):+        column_1    1000000 non-null object+        column_2    1000000 non-null object+        column_3    1000000 non-null object+        dtypes: object(3)+        memory usage: 22.9+ MB++        >>> df.info(memory_usage='deep')+        <class 'pandas.core.frame.DataFrame'>+        RangeIndex: 1000000 entries, 0 to 999999+        Data columns (total 3 columns):+        column_1    1000000 non-null object+        column_2    1000000 non-null object+        column_3    1000000 non-null object+        dtypes: object(3)+        memory usage: 188.8 MB+        """++        if buf is None:  # pragma: no cover+            buf = sys.stdout++        lines = []++        lines.append(str(type(self)))+        lines.append(self.index._summary())++        if len(self.columns) == 0:+            lines.append('Empty {name}'.format(name=type(self).__name__))+            fmt.buffer_put_lines(buf, lines)+            return++        cols = self.columns++        # hack+        if max_cols is None:+            max_cols = get_option('display.max_info_columns',+                                  len(self.columns) + 1)++        max_rows = get_option('display.max_info_rows', len(self) + 1)++        if null_counts is None:+            show_counts = ((len(self.columns) <= max_cols) and+                           (len(self) < max_rows))+        else:+            show_counts = null_counts+        exceeds_info_cols = len(self.columns) > max_cols++        def _verbose_repr():+            lines.append('Data columns (total %d columns):' %+                         len(self.columns))+            space = max(len(pprint_thing(k)) for k in self.columns) + 4+            counts = None++            tmpl = "{count}{dtype}"+            if show_counts:+                counts = self.count()+                if len(cols) != len(counts):  # pragma: no cover+                    raise AssertionError(+                        'Columns must equal counts '+                        '({cols:d} != {counts:d})'.format(+                            cols=len(cols), counts=len(counts)))+                tmpl = "{count} non-null {dtype}"++            dtypes = self.dtypes+            for i, col in enumerate(self.columns):+                dtype = dtypes.iloc[i]+                col = pprint_thing(col)++                count = ""+                if show_counts:+                    count = counts.iloc[i]++                lines.append(_put_str(col, space) + tmpl.format(count=count,+                                                                dtype=dtype))++        def _non_verbose_repr():+            lines.append(self.columns._summary(name='Columns'))++        def _sizeof_fmt(num, size_qualifier):+            # returns size in human readable format+            for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:+                if num < 1024.0:+                    return ("{num:3.1f}{size_q} "+                            "{x}".format(num=num, size_q=size_qualifier, x=x))+                num /= 1024.0+            return "{num:3.1f}{size_q} {pb}".format(num=num,+                                                    size_q=size_qualifier,+                                                    pb='PB')++        if verbose:+            _verbose_repr()+        elif verbose is False:  # specifically set to False, not nesc None+            _non_verbose_repr()+        else:+            if exceeds_info_cols:+                _non_verbose_repr()+            else:+                _verbose_repr()++        counts = self.get_dtype_counts()+        dtypes = ['{k}({kk:d})'.format(k=k[0], kk=k[1]) for k+                  in sorted(compat.iteritems(counts))]+        lines.append('dtypes: {types}'.format(types=', '.join(dtypes)))++        if memory_usage is None:+            memory_usage = get_option('display.memory_usage')+        if memory_usage:+            # append memory usage of df to display+            size_qualifier = ''+            if memory_usage == 'deep':+                deep = True+            else:+                # size_qualifier is just a best effort; not guaranteed to catch+                # all cases (e.g., it misses categorical data even with object+                # categories)+                deep = False+                if ('object' in counts or+                        self.index._is_memory_usage_qualified()):+                    size_qualifier = '+'+            mem_usage = self.memory_usage(index=True, deep=deep).sum()+            lines.append("memory usage: {mem}\n".format(+                mem=_sizeof_fmt(mem_usage, size_qualifier)))++        fmt.buffer_put_lines(buf, lines)++    def memory_usage(self, index=True, deep=False):+        """+        Return the memory usage of each column in bytes.++        The memory usage can optionally include the contribution of+        the index and elements of `object` dtype.++        This value is displayed in `DataFrame.info` by default. This can be+        suppressed by setting ``pandas.options.display.memory_usage`` to False.++        Parameters+        ----------+        index : bool, default True+            Specifies whether to include the memory usage of the DataFrame's+            index in returned Series. If ``index=True`` the memory usage of the+            index the first item in the output.+        deep : bool, default False+            If True, introspect the data deeply by interrogating+            `object` dtypes for system-level memory consumption, and include+            it in the returned values.++        Returns+        -------+        sizes : Series+            A Series whose index is the original column names and whose values+            is the memory usage of each column in bytes.++        See Also+        --------+        numpy.ndarray.nbytes : Total bytes consumed by the elements of an+            ndarray.+        Series.memory_usage : Bytes consumed by a Series.+        pandas.Categorical : Memory-efficient array for string values with+            many repeated values.+        DataFrame.info : Concise summary of a DataFrame.++        Examples+        --------+        >>> dtypes = ['int64', 'float64', 'complex128', 'object', 'bool']+        >>> data = dict([(t, np.ones(shape=5000).astype(t))+        ...              for t in dtypes])+        >>> df = pd.DataFrame(data)+        >>> df.head()+           int64  float64  complex128 object  bool+        0      1      1.0      (1+0j)      1  True+        1      1      1.0      (1+0j)      1  True+        2      1      1.0      (1+0j)      1  True+        3      1      1.0      (1+0j)      1  True+        4      1      1.0      (1+0j)      1  True++        >>> df.memory_usage()+        Index            80+        int64         40000+        float64       40000+        complex128    80000+        object        40000+        bool           5000+        dtype: int64++        >>> df.memory_usage(index=False)+        int64         40000+        float64       40000+        complex128    80000+        object        40000+        bool           5000+        dtype: int64++        The memory footprint of `object` dtype columns is ignored by default:++        >>> df.memory_usage(deep=True)+        Index             80+        int64          40000+        float64        40000+        complex128     80000+        object        160000+        bool            5000+        dtype: int64++        Use a Categorical for efficient storage of an object-dtype column with+        many repeated values.++        >>> df['object'].astype('category').memory_usage(deep=True)+        5168+        """+        result = Series([c.memory_usage(index=False, deep=deep)+                         for col, c in self.iteritems()], index=self.columns)+        if index:+            result = Series(self.index.memory_usage(deep=deep),+                            index=['Index']).append(result)+        return result++    def transpose(self, *args, **kwargs):+        """+        Transpose index and columns.++        Reflect the DataFrame over its main diagonal by writing rows as columns+        and vice-versa. The property :attr:`.T` is an accessor to the method+        :meth:`transpose`.++        Parameters+        ----------+        copy : bool, default False+            If True, the underlying data is copied. Otherwise (default), no+            copy is made if possible.+        *args, **kwargs+            Additional keywords have no effect but might be accepted for+            compatibility with numpy.++        Returns+        -------+        DataFrame+            The transposed DataFrame.++        See Also+        --------+        numpy.transpose : Permute the dimensions of a given array.++        Notes+        -----+        Transposing a DataFrame with mixed dtypes will result in a homogeneous+        DataFrame with the `object` dtype. In such a case, a copy of the data+        is always made.++        Examples+        --------+        **Square DataFrame with homogeneous dtype**++        >>> d1 = {'col1': [1, 2], 'col2': [3, 4]}+        >>> df1 = pd.DataFrame(data=d1)+        >>> df1+           col1  col2+        0     1     3+        1     2     4++        >>> df1_transposed = df1.T # or df1.transpose()+        >>> df1_transposed+              0  1+        col1  1  2+        col2  3  4++        When the dtype is homogeneous in the original DataFrame, we get a+        transposed DataFrame with the same dtype:++        >>> df1.dtypes+        col1    int64+        col2    int64+        dtype: object+        >>> df1_transposed.dtypes+        0    int64+        1    int64+        dtype: object++        **Non-square DataFrame with mixed dtypes**++        >>> d2 = {'name': ['Alice', 'Bob'],+        ...       'score': [9.5, 8],+        ...       'employed': [False, True],+        ...       'kids': [0, 0]}+        >>> df2 = pd.DataFrame(data=d2)+        >>> df2+            name  score  employed  kids+        0  Alice    9.5     False     0+        1    Bob    8.0      True     0++        >>> df2_transposed = df2.T # or df2.transpose()+        >>> df2_transposed+                      0     1+        name      Alice   Bob+        score       9.5     8+        employed  False  True+        kids          0     0++        When the DataFrame has mixed dtypes, we get a transposed DataFrame with+        the `object` dtype:++        >>> df2.dtypes+        name         object+        score       float64+        employed       bool+        kids          int64+        dtype: object+        >>> df2_transposed.dtypes+        0    object+        1    object+        dtype: object+        """+        nv.validate_transpose(args, dict())+        return super(DataFrame, self).transpose(1, 0, **kwargs)++    T = property(transpose)++    # ----------------------------------------------------------------------+    # Picklability++    # legacy pickle formats+    def _unpickle_frame_compat(self, state):  # pragma: no cover+        if len(state) == 2:  # pragma: no cover+            series, idx = state+            columns = sorted(series)+        else:+            series, cols, idx = state+            columns = com._unpickle_array(cols)++        index = com._unpickle_array(idx)+        self._data = self._init_dict(series, index, columns, None)++    def _unpickle_matrix_compat(self, state):  # pragma: no cover+        # old unpickling+        (vals, idx, cols), object_state = state++        index = com._unpickle_array(idx)+        dm = DataFrame(vals, index=index, columns=com._unpickle_array(cols),+                       copy=False)++        if object_state is not None:+            ovals, _, ocols = object_state+            objects = DataFrame(ovals, index=index,+                                columns=com._unpickle_array(ocols), copy=False)++            dm = dm.join(objects)++        self._data = dm._data++    # ----------------------------------------------------------------------+    # Getting and setting elements++    def get_value(self, index, col, takeable=False):+        """Quickly retrieve single value at passed column and index++        .. deprecated:: 0.21.0+            Use .at[] or .iat[] accessors instead.++        Parameters+        ----------+        index : row label+        col : column label+        takeable : interpret the index/col as indexers, default False++        Returns+        -------+        value : scalar value+        """++        warnings.warn("get_value is deprecated and will be removed "+                      "in a future release. Please use "+                      ".at[] or .iat[] accessors instead", FutureWarning,+                      stacklevel=2)+        return self._get_value(index, col, takeable=takeable)++    def _get_value(self, index, col, takeable=False):++        if takeable:+            series = self._iget_item_cache(col)+            return com.maybe_box_datetimelike(series._values[index])++        series = self._get_item_cache(col)+        engine = self.index._engine++        try:+            return engine.get_value(series._values, index)+        except (TypeError, ValueError):++            # we cannot handle direct indexing+            # use positional+            col = self.columns.get_loc(col)+            index = self.index.get_loc(index)+            return self._get_value(index, col, takeable=True)+    _get_value.__doc__ = get_value.__doc__++    def set_value(self, index, col, value, takeable=False):+        """Put single value at passed column and index++        .. deprecated:: 0.21.0+            Use .at[] or .iat[] accessors instead.++        Parameters+        ----------+        index : row label+        col : column label+        value : scalar value+        takeable : interpret the index/col as indexers, default False++        Returns+        -------+        frame : DataFrame+            If label pair is contained, will be reference to calling DataFrame,+            otherwise a new object+        """+        warnings.warn("set_value is deprecated and will be removed "+                      "in a future release. Please use "+                      ".at[] or .iat[] accessors instead", FutureWarning,+                      stacklevel=2)+        return self._set_value(index, col, value, takeable=takeable)++    def _set_value(self, index, col, value, takeable=False):+        try:+            if takeable is True:+                series = self._iget_item_cache(col)+                return series._set_value(index, value, takeable=True)++            series = self._get_item_cache(col)+            engine = self.index._engine+            engine.set_value(series._values, index, value)+            return self+        except (KeyError, TypeError):++            # set using a non-recursive method & reset the cache+            self.loc[index, col] = value+            self._item_cache.pop(col, None)++            return self+    _set_value.__doc__ = set_value.__doc__++    def _ixs(self, i, axis=0):+        """+        i : int, slice, or sequence of integers+        axis : int+        """++        # irow+        if axis == 0:+            """+            Notes+            -----+            If slice passed, the resulting data will be a view+            """++            if isinstance(i, slice):+                return self[i]+            else:+                label = self.index[i]+                if isinstance(label, Index):+                    # a location index by definition+                    result = self.take(i, axis=axis)+                    copy = True+                else:+                    new_values = self._data.fast_xs(i)+                    if is_scalar(new_values):+                        return new_values++                    # if we are a copy, mark as such+                    copy = (isinstance(new_values, np.ndarray) and+                            new_values.base is None)+                    result = self._constructor_sliced(new_values,+                                                      index=self.columns,+                                                      name=self.index[i],+                                                      dtype=new_values.dtype)+                result._set_is_copy(self, copy=copy)+                return result++        # icol+        else:+            """+            Notes+            -----+            If slice passed, the resulting data will be a view+            """++            label = self.columns[i]+            if isinstance(i, slice):+                # need to return view+                lab_slice = slice(label[0], label[-1])+                return self.loc[:, lab_slice]+            else:+                if isinstance(label, Index):+                    return self._take(i, axis=1)++                index_len = len(self.index)++                # if the values returned are not the same length+                # as the index (iow a not found value), iget returns+                # a 0-len ndarray. This is effectively catching+                # a numpy error (as numpy should really raise)+                values = self._data.iget(i)++                if index_len and not len(values):+                    values = np.array([np.nan] * index_len, dtype=object)+                result = self._box_col_values(values, label)++                # this is a cached value, mark it so+                result._set_as_cached(label, self)++                return result++    def __getitem__(self, key):+        key = com.apply_if_callable(key, self)++        # shortcut if the key is in columns+        try:+            if self.columns.is_unique and key in self.columns:+                if self.columns.nlevels > 1:+                    return self._getitem_multilevel(key)+                return self._get_item_cache(key)+        except (TypeError, ValueError):+            # The TypeError correctly catches non hashable "key" (e.g. list)+            # The ValueError can be removed once GH #21729 is fixed+            pass++        # Do we have a slicer (on rows)?+        indexer = convert_to_index_sliceable(self, key)+        if indexer is not None:+            return self._slice(indexer, axis=0)++        # Do we have a (boolean) DataFrame?+        if isinstance(key, DataFrame):+            return self._getitem_frame(key)++        # Do we have a (boolean) 1d indexer?+        if com.is_bool_indexer(key):+            return self._getitem_bool_array(key)++        # We are left with two options: a single key, and a collection of keys,+        # We interpret tuples as collections only for non-MultiIndex+        is_single_key = isinstance(key, tuple) or not is_list_like(key)++        if is_single_key:+            if self.columns.nlevels > 1:+                return self._getitem_multilevel(key)+            indexer = self.columns.get_loc(key)+            if is_integer(indexer):+                indexer = [indexer]+        else:+            if is_iterator(key):+                key = list(key)+            indexer = self.loc._convert_to_indexer(key, axis=1,+                                                   raise_missing=True)++        # take() does not accept boolean indexers+        if getattr(indexer, "dtype", None) == bool:+            indexer = np.where(indexer)[0]++        data = self._take(indexer, axis=1)++        if is_single_key:+            # What does looking for a single key in a non-unique index return?+            # The behavior is inconsistent. It returns a Series, except when+            # - the key itself is repeated (test on data.shape, #9519), or+            # - we have a MultiIndex on columns (test on self.columns, #21309)+            if data.shape[1] == 1 and not isinstance(self.columns, MultiIndex):+                data = data[key]++        return data++    def _getitem_bool_array(self, key):+        # also raises Exception if object array with NA values+        # warning here just in case -- previously __setitem__ was+        # reindexing but __getitem__ was not; it seems more reasonable to+        # go with the __setitem__ behavior since that is more consistent+        # with all other indexing behavior+        if isinstance(key, Series) and not key.index.equals(self.index):+            warnings.warn("Boolean Series key will be reindexed to match "+                          "DataFrame index.", UserWarning, stacklevel=3)+        elif len(key) != len(self.index):+            raise ValueError('Item wrong length %d instead of %d.' %+                             (len(key), len(self.index)))++        # check_bool_indexer will throw exception if Series key cannot+        # be reindexed to match DataFrame rows+        key = check_bool_indexer(self.index, key)+        indexer = key.nonzero()[0]+        return self._take(indexer, axis=0)++    def _getitem_multilevel(self, key):+        loc = self.columns.get_loc(key)+        if isinstance(loc, (slice, Series, np.ndarray, Index)):+            new_columns = self.columns[loc]+            result_columns = maybe_droplevels(new_columns, key)+            if self._is_mixed_type:+                result = self.reindex(columns=new_columns)+                result.columns = result_columns+            else:+                new_values = self.values[:, loc]+                result = self._constructor(new_values, index=self.index,+                                           columns=result_columns)+                result = result.__finalize__(self)++            # If there is only one column being returned, and its name is+            # either an empty string, or a tuple with an empty string as its+            # first element, then treat the empty string as a placeholder+            # and return the column as if the user had provided that empty+            # string in the key. If the result is a Series, exclude the+            # implied empty string from its name.+            if len(result.columns) == 1:+                top = result.columns[0]+                if isinstance(top, tuple):+                    top = top[0]+                if top == '':+                    result = result['']+                    if isinstance(result, Series):+                        result = self._constructor_sliced(result,+                                                          index=self.index,+                                                          name=key)++            result._set_is_copy(self)+            return result+        else:+            return self._get_item_cache(key)++    def _getitem_frame(self, key):+        if key.values.size and not is_bool_dtype(key.values):+            raise ValueError('Must pass DataFrame with boolean values only')+        return self.where(key)++    def query(self, expr, inplace=False, **kwargs):+        """Query the columns of a frame with a boolean expression.++        Parameters+        ----------+        expr : string+            The query string to evaluate.  You can refer to variables+            in the environment by prefixing them with an '@' character like+            ``@a + b``.+        inplace : bool+            Whether the query should modify the data in place or return+            a modified copy++            .. versionadded:: 0.18.0++        kwargs : dict+            See the documentation for :func:`pandas.eval` for complete details+            on the keyword arguments accepted by :meth:`DataFrame.query`.++        Returns+        -------+        q : DataFrame++        Notes+        -----+        The result of the evaluation of this expression is first passed to+        :attr:`DataFrame.loc` and if that fails because of a+        multidimensional key (e.g., a DataFrame) then the result will be passed+        to :meth:`DataFrame.__getitem__`.++        This method uses the top-level :func:`pandas.eval` function to+        evaluate the passed query.++        The :meth:`~pandas.DataFrame.query` method uses a slightly+        modified Python syntax by default. For example, the ``&`` and ``|``+        (bitwise) operators have the precedence of their boolean cousins,+        :keyword:`and` and :keyword:`or`. This *is* syntactically valid Python,+        however the semantics are different.++        You can change the semantics of the expression by passing the keyword+        argument ``parser='python'``. This enforces the same semantics as+        evaluation in Python space. Likewise, you can pass ``engine='python'``+        to evaluate an expression using Python itself as a backend. This is not+        recommended as it is inefficient compared to using ``numexpr`` as the+        engine.++        The :attr:`DataFrame.index` and+        :attr:`DataFrame.columns` attributes of the+        :class:`~pandas.DataFrame` instance are placed in the query namespace+        by default, which allows you to treat both the index and columns of the+        frame as a column in the frame.+        The identifier ``index`` is used for the frame index; you can also+        use the name of the index to identify it in a query. Please note that+        Python keywords may not be used as identifiers.++        For further details and examples see the ``query`` documentation in+        :ref:`indexing <indexing.query>`.++        See Also+        --------+        pandas.eval+        DataFrame.eval++        Examples+        --------+        >>> df = pd.DataFrame(np.random.randn(10, 2), columns=list('ab'))+        >>> df.query('a > b')+        >>> df[df.a > df.b]  # same result as the previous expression+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        if not isinstance(expr, compat.string_types):+            msg = "expr must be a string to be evaluated, {0} given"+            raise ValueError(msg.format(type(expr)))+        kwargs['level'] = kwargs.pop('level', 0) + 1+        kwargs['target'] = None+        res = self.eval(expr, **kwargs)++        try:+            new_data = self.loc[res]+        except ValueError:+            # when res is multi-dimensional loc raises, but this is sometimes a+            # valid query+            new_data = self[res]++        if inplace:+            self._update_inplace(new_data)+        else:+            return new_data++    def eval(self, expr, inplace=False, **kwargs):+        """+        Evaluate a string describing operations on DataFrame columns.++        Operates on columns only, not specific rows or elements.  This allows+        `eval` to run arbitrary code, which can make you vulnerable to code+        injection if you pass user input to this function.++        Parameters+        ----------+        expr : str+            The expression string to evaluate.+        inplace : bool, default False+            If the expression contains an assignment, whether to perform the+            operation inplace and mutate the existing DataFrame. Otherwise,+            a new DataFrame is returned.++            .. versionadded:: 0.18.0.+        kwargs : dict+            See the documentation for :func:`~pandas.eval` for complete details+            on the keyword arguments accepted by+            :meth:`~pandas.DataFrame.query`.++        Returns+        -------+        ndarray, scalar, or pandas object+            The result of the evaluation.++        See Also+        --------+        DataFrame.query : Evaluates a boolean expression to query the columns+            of a frame.+        DataFrame.assign : Can evaluate an expression or function to create new+            values for a column.+        pandas.eval : Evaluate a Python expression as a string using various+            backends.++        Notes+        -----+        For more details see the API documentation for :func:`~pandas.eval`.+        For detailed examples see :ref:`enhancing performance with eval+        <enhancingperf.eval>`.++        Examples+        --------+        >>> df = pd.DataFrame({'A': range(1, 6), 'B': range(10, 0, -2)})+        >>> df+           A   B+        0  1  10+        1  2   8+        2  3   6+        3  4   4+        4  5   2+        >>> df.eval('A + B')+        0    11+        1    10+        2     9+        3     8+        4     7+        dtype: int64++        Assignment is allowed though by default the original DataFrame is not+        modified.++        >>> df.eval('C = A + B')+           A   B   C+        0  1  10  11+        1  2   8  10+        2  3   6   9+        3  4   4   8+        4  5   2   7+        >>> df+           A   B+        0  1  10+        1  2   8+        2  3   6+        3  4   4+        4  5   2++        Use ``inplace=True`` to modify the original DataFrame.++        >>> df.eval('C = A + B', inplace=True)+        >>> df+           A   B   C+        0  1  10  11+        1  2   8  10+        2  3   6   9+        3  4   4   8+        4  5   2   7+        """+        from pandas.core.computation.eval import eval as _eval++        inplace = validate_bool_kwarg(inplace, 'inplace')+        resolvers = kwargs.pop('resolvers', None)+        kwargs['level'] = kwargs.pop('level', 0) + 1+        if resolvers is None:+            index_resolvers = self._get_index_resolvers()+            resolvers = dict(self.iteritems()), index_resolvers+        if 'target' not in kwargs:+            kwargs['target'] = self+        kwargs['resolvers'] = kwargs.get('resolvers', ()) + tuple(resolvers)+        return _eval(expr, inplace=inplace, **kwargs)++    def select_dtypes(self, include=None, exclude=None):+        """+        Return a subset of the DataFrame's columns based on the column dtypes.++        Parameters+        ----------+        include, exclude : scalar or list-like+            A selection of dtypes or strings to be included/excluded. At least+            one of these parameters must be supplied.++        Raises+        ------+        ValueError+            * If both of ``include`` and ``exclude`` are empty+            * If ``include`` and ``exclude`` have overlapping elements+            * If any kind of string dtype is passed in.++        Returns+        -------+        subset : DataFrame+            The subset of the frame including the dtypes in ``include`` and+            excluding the dtypes in ``exclude``.++        Notes+        -----+        * To select all *numeric* types, use ``np.number`` or ``'number'``+        * To select strings you must use the ``object`` dtype, but note that+          this will return *all* object dtype columns+        * See the `numpy dtype hierarchy+          <http://docs.scipy.org/doc/numpy/reference/arrays.scalars.html>`__+        * To select datetimes, use ``np.datetime64``, ``'datetime'`` or+          ``'datetime64'``+        * To select timedeltas, use ``np.timedelta64``, ``'timedelta'`` or+          ``'timedelta64'``+        * To select Pandas categorical dtypes, use ``'category'``+        * To select Pandas datetimetz dtypes, use ``'datetimetz'`` (new in+          0.20.0) or ``'datetime64[ns, tz]'``++        Examples+        --------+        >>> df = pd.DataFrame({'a': [1, 2] * 3,+        ...                    'b': [True, False] * 3,+        ...                    'c': [1.0, 2.0] * 3})+        >>> df+                a      b  c+        0       1   True  1.0+        1       2  False  2.0+        2       1   True  1.0+        3       2  False  2.0+        4       1   True  1.0+        5       2  False  2.0++        >>> df.select_dtypes(include='bool')+           b+        0  True+        1  False+        2  True+        3  False+        4  True+        5  False++        >>> df.select_dtypes(include=['float64'])+           c+        0  1.0+        1  2.0+        2  1.0+        3  2.0+        4  1.0+        5  2.0++        >>> df.select_dtypes(exclude=['int'])+               b    c+        0   True  1.0+        1  False  2.0+        2   True  1.0+        3  False  2.0+        4   True  1.0+        5  False  2.0+        """++        if not is_list_like(include):+            include = (include,) if include is not None else ()+        if not is_list_like(exclude):+            exclude = (exclude,) if exclude is not None else ()++        selection = tuple(map(frozenset, (include, exclude)))++        if not any(selection):+            raise ValueError('at least one of include or exclude must be '+                             'nonempty')++        # convert the myriad valid dtypes object to a single representation+        include, exclude = map(+            lambda x: frozenset(map(_get_dtype_from_object, x)), selection)+        for dtypes in (include, exclude):+            invalidate_string_dtypes(dtypes)++        # can't both include AND exclude!+        if not include.isdisjoint(exclude):+            raise ValueError('include and exclude overlap on {inc_ex}'.format(+                inc_ex=(include & exclude)))++        # empty include/exclude -> defaults to True+        # three cases (we've already raised if both are empty)+        # case 1: empty include, nonempty exclude+        # we have True, True, ... True for include, same for exclude+        # in the loop below we get the excluded+        # and when we call '&' below we get only the excluded+        # case 2: nonempty include, empty exclude+        # same as case 1, but with include+        # case 3: both nonempty+        # the "union" of the logic of case 1 and case 2:+        # we get the included and excluded, and return their logical and+        include_these = Series(not bool(include), index=self.columns)+        exclude_these = Series(not bool(exclude), index=self.columns)++        def is_dtype_instance_mapper(idx, dtype):+            return idx, functools.partial(issubclass, dtype.type)++        for idx, f in itertools.starmap(is_dtype_instance_mapper,+                                        enumerate(self.dtypes)):+            if include:  # checks for the case of empty include or exclude+                include_these.iloc[idx] = any(map(f, include))+            if exclude:+                exclude_these.iloc[idx] = not any(map(f, exclude))++        dtype_indexer = include_these & exclude_these+        return self.loc[com.get_info_slice(self, dtype_indexer)]++    def _box_item_values(self, key, values):+        items = self.columns[self.columns.get_loc(key)]+        if values.ndim == 2:+            return self._constructor(values.T, columns=items, index=self.index)+        else:+            return self._box_col_values(values, items)++    def _box_col_values(self, values, items):+        """ provide boxed values for a column """+        klass = _get_sliced_frame_result_type(values, self)+        return klass(values, index=self.index, name=items, fastpath=True)++    def __setitem__(self, key, value):+        key = com.apply_if_callable(key, self)++        # see if we can slice the rows+        indexer = convert_to_index_sliceable(self, key)+        if indexer is not None:+            return self._setitem_slice(indexer, value)++        if isinstance(key, DataFrame) or getattr(key, 'ndim', None) == 2:+            self._setitem_frame(key, value)+        elif isinstance(key, (Series, np.ndarray, list, Index)):+            self._setitem_array(key, value)+        else:+            # set column+            self._set_item(key, value)++    def _setitem_slice(self, key, value):+        self._check_setitem_copy()+        self.loc._setitem_with_indexer(key, value)++    def _setitem_array(self, key, value):+        # also raises Exception if object array with NA values+        if com.is_bool_indexer(key):+            if len(key) != len(self.index):+                raise ValueError('Item wrong length %d instead of %d!' %+                                 (len(key), len(self.index)))+            key = check_bool_indexer(self.index, key)+            indexer = key.nonzero()[0]+            self._check_setitem_copy()+            self.loc._setitem_with_indexer(indexer, value)+        else:+            if isinstance(value, DataFrame):+                if len(value.columns) != len(key):+                    raise ValueError('Columns must be same length as key')+                for k1, k2 in zip(key, value.columns):+                    self[k1] = value[k2]+            else:+                indexer = self.loc._convert_to_indexer(key, axis=1)+                self._check_setitem_copy()+                self.loc._setitem_with_indexer((slice(None), indexer), value)++    def _setitem_frame(self, key, value):+        # support boolean setting with DataFrame input, e.g.+        # df[df > df2] = 0+        if isinstance(key, np.ndarray):+            if key.shape != self.shape:+                raise ValueError(+                    'Array conditional must be same shape as self'+                )+            key = self._constructor(key, **self._construct_axes_dict())++        if key.values.size and not is_bool_dtype(key.values):+            raise TypeError(+                'Must pass DataFrame or 2-d ndarray with boolean values only'+            )++        self._check_inplace_setting(value)+        self._check_setitem_copy()+        self._where(-key, value, inplace=True)++    def _ensure_valid_index(self, value):+        """+        ensure that if we don't have an index, that we can create one from the+        passed value+        """+        # GH5632, make sure that we are a Series convertible+        if not len(self.index) and is_list_like(value):+            try:+                value = Series(value)+            except:+                raise ValueError('Cannot set a frame with no defined index '+                                 'and a value that cannot be converted to a '+                                 'Series')++            self._data = self._data.reindex_axis(value.index.copy(), axis=1,+                                                 fill_value=np.nan)++    def _set_item(self, key, value):+        """+        Add series to DataFrame in specified column.++        If series is a numpy-array (not a Series/TimeSeries), it must be the+        same length as the DataFrames index or an error will be thrown.++        Series/TimeSeries will be conformed to the DataFrames index to+        ensure homogeneity.+        """++        self._ensure_valid_index(value)+        value = self._sanitize_column(key, value)+        NDFrame._set_item(self, key, value)++        # check if we are modifying a copy+        # try to set first as we want an invalid+        # value exception to occur first+        if len(self):+            self._check_setitem_copy()++    def insert(self, loc, column, value, allow_duplicates=False):+        """+        Insert column into DataFrame at specified location.++        Raises a ValueError if `column` is already contained in the DataFrame,+        unless `allow_duplicates` is set to True.++        Parameters+        ----------+        loc : int+            Insertion index. Must verify 0 <= loc <= len(columns)+        column : string, number, or hashable object+            label of the inserted column+        value : int, Series, or array-like+        allow_duplicates : bool, optional+        """+        self._ensure_valid_index(value)+        value = self._sanitize_column(column, value, broadcast=False)+        self._data.insert(loc, column, value,+                          allow_duplicates=allow_duplicates)++    def assign(self, **kwargs):+        r"""+        Assign new columns to a DataFrame.++        Returns a new object with all original columns in addition to new ones.+        Existing columns that are re-assigned will be overwritten.++        Parameters+        ----------+        kwargs : keyword, value pairs+            The column names are keywords. If the values are+            callable, they are computed on the DataFrame and+            assigned to the new columns. The callable must not+            change input DataFrame (though pandas doesn't check it).+            If the values are not callable, (e.g. a Series, scalar, or array),+            they are simply assigned.++        Returns+        -------+        df : DataFrame+            A new DataFrame with the new columns in addition to+            all the existing columns.++        Notes+        -----+        Assigning multiple columns within the same ``assign`` is possible.+        For Python 3.6 and above, later items in '\*\*kwargs' may refer to+        newly created or modified columns in 'df'; items are computed and+        assigned into 'df' in order.  For Python 3.5 and below, the order of+        keyword arguments is not specified, you cannot refer to newly created+        or modified columns. All items are computed first, and then assigned+        in alphabetical order.++        .. versionchanged :: 0.23.0++           Keyword argument order is maintained for Python 3.6 and later.++        Examples+        --------+        >>> df = pd.DataFrame({'A': range(1, 11), 'B': np.random.randn(10)})++        Where the value is a callable, evaluated on `df`:++        >>> df.assign(ln_A = lambda x: np.log(x.A))+            A         B      ln_A+        0   1  0.426905  0.000000+        1   2 -0.780949  0.693147+        2   3 -0.418711  1.098612+        3   4 -0.269708  1.386294+        4   5 -0.274002  1.609438+        5   6 -0.500792  1.791759+        6   7  1.649697  1.945910+        7   8 -1.495604  2.079442+        8   9  0.549296  2.197225+        9  10 -0.758542  2.302585++        Where the value already exists and is inserted:++        >>> newcol = np.log(df['A'])+        >>> df.assign(ln_A=newcol)+            A         B      ln_A+        0   1  0.426905  0.000000+        1   2 -0.780949  0.693147+        2   3 -0.418711  1.098612+        3   4 -0.269708  1.386294+        4   5 -0.274002  1.609438+        5   6 -0.500792  1.791759+        6   7  1.649697  1.945910+        7   8 -1.495604  2.079442+        8   9  0.549296  2.197225+        9  10 -0.758542  2.302585++        Where the keyword arguments depend on each other++        >>> df = pd.DataFrame({'A': [1, 2, 3]})++        >>> df.assign(B=df.A, C=lambda x:x['A']+ x['B'])+            A  B  C+         0  1  1  2+         1  2  2  4+         2  3  3  6+        """+        data = self.copy()++        # >= 3.6 preserve order of kwargs+        if PY36:+            for k, v in kwargs.items():+                data[k] = com.apply_if_callable(v, data)+        else:+            # <= 3.5: do all calculations first...+            results = OrderedDict()+            for k, v in kwargs.items():+                results[k] = com.apply_if_callable(v, data)++            # <= 3.5 and earlier+            results = sorted(results.items())+            # ... and then assign+            for k, v in results:+                data[k] = v+        return data++    def _sanitize_column(self, key, value, broadcast=True):+        """+        Ensures new columns (which go into the BlockManager as new blocks) are+        always copied and converted into an array.++        Parameters+        ----------+        key : object+        value : scalar, Series, or array-like+        broadcast : bool, default True+            If ``key`` matches multiple duplicate column names in the+            DataFrame, this parameter indicates whether ``value`` should be+            tiled so that the returned array contains a (duplicated) column for+            each occurrence of the key. If False, ``value`` will not be tiled.++        Returns+        -------+        sanitized_column : numpy-array+        """++        def reindexer(value):+            # reindex if necessary++            if value.index.equals(self.index) or not len(self.index):+                value = value._values.copy()+            else:++                # GH 4107+                try:+                    value = value.reindex(self.index)._values+                except Exception as e:++                    # duplicate axis+                    if not value.index.is_unique:+                        raise e++                    # other+                    raise TypeError('incompatible index of inserted column '+                                    'with frame index')+            return value++        if isinstance(value, Series):+            value = reindexer(value)++        elif isinstance(value, DataFrame):+            # align right-hand-side columns if self.columns+            # is multi-index and self[key] is a sub-frame+            if isinstance(self.columns, MultiIndex) and key in self.columns:+                loc = self.columns.get_loc(key)+                if isinstance(loc, (slice, Series, np.ndarray, Index)):+                    cols = maybe_droplevels(self.columns[loc], key)+                    if len(cols) and not cols.equals(value.columns):+                        value = value.reindex(cols, axis=1)+            # now align rows+            value = reindexer(value).T++        elif isinstance(value, ExtensionArray):+            from pandas.core.series import _sanitize_index+            # Explicitly copy here, instead of in _sanitize_index,+            # as sanitize_index won't copy an EA, even with copy=True+            value = value.copy()+            value = _sanitize_index(value, self.index, copy=False)++        elif isinstance(value, Index) or is_sequence(value):+            from pandas.core.series import _sanitize_index++            # turn me into an ndarray+            value = _sanitize_index(value, self.index, copy=False)+            if not isinstance(value, (np.ndarray, Index)):+                if isinstance(value, list) and len(value) > 0:+                    value = maybe_convert_platform(value)+                else:+                    value = com.asarray_tuplesafe(value)+            elif value.ndim == 2:+                value = value.copy().T+            elif isinstance(value, Index):+                value = value.copy(deep=True)+            else:+                value = value.copy()++            # possibly infer to datetimelike+            if is_object_dtype(value.dtype):+                value = maybe_infer_to_datetimelike(value)++        else:+            # cast ignores pandas dtypes. so save the dtype first+            infer_dtype, _ = infer_dtype_from_scalar(+                value, pandas_dtype=True)++            # upcast+            value = cast_scalar_to_array(len(self.index), value)+            value = maybe_cast_to_datetime(value, infer_dtype)++        # return internal types directly+        if is_extension_type(value) or is_extension_array_dtype(value):+            return value++        # broadcast across multiple columns if necessary+        if broadcast and key in self.columns and value.ndim == 1:+            if (not self.columns.is_unique or+                    isinstance(self.columns, MultiIndex)):+                existing_piece = self[key]+                if isinstance(existing_piece, DataFrame):+                    value = np.tile(value, (len(existing_piece.columns), 1))++        return np.atleast_2d(np.asarray(value))++    @property+    def _series(self):+        result = {}+        for idx, item in enumerate(self.columns):+            result[item] = Series(self._data.iget(idx), index=self.index,+                                  name=item)+        return result++    def lookup(self, row_labels, col_labels):+        """Label-based "fancy indexing" function for DataFrame.+        Given equal-length arrays of row and column labels, return an+        array of the values corresponding to each (row, col) pair.++        Parameters+        ----------+        row_labels : sequence+            The row labels to use for lookup+        col_labels : sequence+            The column labels to use for lookup++        Notes+        -----+        Akin to::++            result = []+            for row, col in zip(row_labels, col_labels):+                result.append(df.get_value(row, col))++        Examples+        --------+        values : ndarray+            The found values++        """+        n = len(row_labels)+        if n != len(col_labels):+            raise ValueError('Row labels must have same size as column labels')++        thresh = 1000+        if not self._is_mixed_type or n > thresh:+            values = self.values+            ridx = self.index.get_indexer(row_labels)+            cidx = self.columns.get_indexer(col_labels)+            if (ridx == -1).any():+                raise KeyError('One or more row labels was not found')+            if (cidx == -1).any():+                raise KeyError('One or more column labels was not found')+            flat_index = ridx * len(self.columns) + cidx+            result = values.flat[flat_index]+        else:+            result = np.empty(n, dtype='O')+            for i, (r, c) in enumerate(zip(row_labels, col_labels)):+                result[i] = self._get_value(r, c)++        if is_object_dtype(result):+            result = lib.maybe_convert_objects(result)++        return result++    # ----------------------------------------------------------------------+    # Reindexing and alignment++    def _reindex_axes(self, axes, level, limit, tolerance, method, fill_value,+                      copy):+        frame = self++        columns = axes['columns']+        if columns is not None:+            frame = frame._reindex_columns(columns, method, copy, level,+                                           fill_value, limit, tolerance)++        index = axes['index']+        if index is not None:+            frame = frame._reindex_index(index, method, copy, level,+                                         fill_value, limit, tolerance)++        return frame++    def _reindex_index(self, new_index, method, copy, level, fill_value=np.nan,+                       limit=None, tolerance=None):+        new_index, indexer = self.index.reindex(new_index, method=method,+                                                level=level, limit=limit,+                                                tolerance=tolerance)+        return self._reindex_with_indexers({0: [new_index, indexer]},+                                           copy=copy, fill_value=fill_value,+                                           allow_dups=False)++    def _reindex_columns(self, new_columns, method, copy, level,+                         fill_value=None, limit=None, tolerance=None):+        new_columns, indexer = self.columns.reindex(new_columns, method=method,+                                                    level=level, limit=limit,+                                                    tolerance=tolerance)+        return self._reindex_with_indexers({1: [new_columns, indexer]},+                                           copy=copy, fill_value=fill_value,+                                           allow_dups=False)++    def _reindex_multi(self, axes, copy, fill_value):+        """ we are guaranteed non-Nones in the axes! """++        new_index, row_indexer = self.index.reindex(axes['index'])+        new_columns, col_indexer = self.columns.reindex(axes['columns'])++        if row_indexer is not None and col_indexer is not None:+            indexer = row_indexer, col_indexer+            new_values = algorithms.take_2d_multi(self.values, indexer,+                                                  fill_value=fill_value)+            return self._constructor(new_values, index=new_index,+                                     columns=new_columns)+        else:+            return self._reindex_with_indexers({0: [new_index, row_indexer],+                                                1: [new_columns, col_indexer]},+                                               copy=copy,+                                               fill_value=fill_value)++    @Appender(_shared_docs['align'] % _shared_doc_kwargs)+    def align(self, other, join='outer', axis=None, level=None, copy=True,+              fill_value=None, method=None, limit=None, fill_axis=0,+              broadcast_axis=None):+        return super(DataFrame, self).align(other, join=join, axis=axis,+                                            level=level, copy=copy,+                                            fill_value=fill_value,+                                            method=method, limit=limit,+                                            fill_axis=fill_axis,+                                            broadcast_axis=broadcast_axis)++    @Appender(_shared_docs['reindex'] % _shared_doc_kwargs)+    @rewrite_axis_style_signature('labels', [('method', None),+                                             ('copy', True),+                                             ('level', None),+                                             ('fill_value', np.nan),+                                             ('limit', None),+                                             ('tolerance', None)])+    def reindex(self, *args, **kwargs):+        axes = validate_axis_style_args(self, args, kwargs, 'labels',+                                        'reindex')+        kwargs.update(axes)+        # Pop these, since the values are in `kwargs` under different names+        kwargs.pop('axis', None)+        kwargs.pop('labels', None)+        return super(DataFrame, self).reindex(**kwargs)++    @Appender(_shared_docs['reindex_axis'] % _shared_doc_kwargs)+    def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True,+                     limit=None, fill_value=np.nan):+        return super(DataFrame,+                     self).reindex_axis(labels=labels, axis=axis,+                                        method=method, level=level, copy=copy,+                                        limit=limit, fill_value=fill_value)++    def drop(self, labels=None, axis=0, index=None, columns=None,+             level=None, inplace=False, errors='raise'):+        """+        Drop specified labels from rows or columns.++        Remove rows or columns by specifying label names and corresponding+        axis, or by specifying directly index or column names. When using a+        multi-index, labels on different levels can be removed by specifying+        the level.++        Parameters+        ----------+        labels : single label or list-like+            Index or column labels to drop.+        axis : {0 or 'index', 1 or 'columns'}, default 0+            Whether to drop labels from the index (0 or 'index') or+            columns (1 or 'columns').+        index, columns : single label or list-like+            Alternative to specifying axis (``labels, axis=1``+            is equivalent to ``columns=labels``).++            .. versionadded:: 0.21.0+        level : int or level name, optional+            For MultiIndex, level from which the labels will be removed.+        inplace : bool, default False+            If True, do operation inplace and return None.+        errors : {'ignore', 'raise'}, default 'raise'+            If 'ignore', suppress error and only existing labels are+            dropped.++        Returns+        -------+        dropped : pandas.DataFrame++        See Also+        --------+        DataFrame.loc : Label-location based indexer for selection by label.+        DataFrame.dropna : Return DataFrame with labels on given axis omitted+            where (all or any) data are missing+        DataFrame.drop_duplicates : Return DataFrame with duplicate rows+            removed, optionally only considering certain columns+        Series.drop : Return Series with specified index labels removed.++        Raises+        ------+        KeyError+            If none of the labels are found in the selected axis++        Examples+        --------+        >>> df = pd.DataFrame(np.arange(12).reshape(3,4),+        ...                   columns=['A', 'B', 'C', 'D'])+        >>> df+           A  B   C   D+        0  0  1   2   3+        1  4  5   6   7+        2  8  9  10  11++        Drop columns++        >>> df.drop(['B', 'C'], axis=1)+           A   D+        0  0   3+        1  4   7+        2  8  11++        >>> df.drop(columns=['B', 'C'])+           A   D+        0  0   3+        1  4   7+        2  8  11++        Drop a row by index++        >>> df.drop([0, 1])+           A  B   C   D+        2  8  9  10  11++        Drop columns and/or rows of MultiIndex DataFrame++        >>> midx = pd.MultiIndex(levels=[['lama', 'cow', 'falcon'],+        ...                              ['speed', 'weight', 'length']],+        ...                      labels=[[0, 0, 0, 1, 1, 1, 2, 2, 2],+        ...                              [0, 1, 2, 0, 1, 2, 0, 1, 2]])+        >>> df = pd.DataFrame(index=midx, columns=['big', 'small'],+        ...                   data=[[45, 30], [200, 100], [1.5, 1], [30, 20],+        ...                         [250, 150], [1.5, 0.8], [320, 250],+        ...                         [1, 0.8], [0.3,0.2]])+        >>> df+                        big     small+        lama    speed   45.0    30.0+                weight  200.0   100.0+                length  1.5     1.0+        cow     speed   30.0    20.0+                weight  250.0   150.0+                length  1.5     0.8+        falcon  speed   320.0   250.0+                weight  1.0     0.8+                length  0.3     0.2++        >>> df.drop(index='cow', columns='small')+                        big+        lama    speed   45.0+                weight  200.0+                length  1.5+        falcon  speed   320.0+                weight  1.0+                length  0.3++        >>> df.drop(index='length', level=1)+                        big     small+        lama    speed   45.0    30.0+                weight  200.0   100.0+        cow     speed   30.0    20.0+                weight  250.0   150.0+        falcon  speed   320.0   250.0+                weight  1.0     0.8+        """+        return super(DataFrame, self).drop(labels=labels, axis=axis,+                                           index=index, columns=columns,+                                           level=level, inplace=inplace,+                                           errors=errors)++    @rewrite_axis_style_signature('mapper', [('copy', True),+                                             ('inplace', False),+                                             ('level', None)])+    def rename(self, *args, **kwargs):+        """Alter axes labels.++        Function / dict values must be unique (1-to-1). Labels not contained in+        a dict / Series will be left as-is. Extra labels listed don't throw an+        error.++        See the :ref:`user guide <basics.rename>` for more.++        Parameters+        ----------+        mapper, index, columns : dict-like or function, optional+            dict-like or functions transformations to apply to+            that axis' values. Use either ``mapper`` and ``axis`` to+            specify the axis to target with ``mapper``, or ``index`` and+            ``columns``.+        axis : int or str, optional+            Axis to target with ``mapper``. Can be either the axis name+            ('index', 'columns') or number (0, 1). The default is 'index'.+        copy : boolean, default True+            Also copy underlying data+        inplace : boolean, default False+            Whether to return a new DataFrame. If True then value of copy is+            ignored.+        level : int or level name, default None+            In case of a MultiIndex, only rename labels in the specified+            level.++        Returns+        -------+        renamed : DataFrame++        See Also+        --------+        pandas.DataFrame.rename_axis++        Examples+        --------++        ``DataFrame.rename`` supports two calling conventions++        * ``(index=index_mapper, columns=columns_mapper, ...)``+        * ``(mapper, axis={'index', 'columns'}, ...)``++        We *highly* recommend using keyword arguments to clarify your+        intent.++        >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})+        >>> df.rename(index=str, columns={"A": "a", "B": "c"})+           a  c+        0  1  4+        1  2  5+        2  3  6++        >>> df.rename(index=str, columns={"A": "a", "C": "c"})+           a  B+        0  1  4+        1  2  5+        2  3  6++        Using axis-style parameters++        >>> df.rename(str.lower, axis='columns')+           a  b+        0  1  4+        1  2  5+        2  3  6++        >>> df.rename({1: 2, 2: 4}, axis='index')+           A  B+        0  1  4+        2  2  5+        4  3  6+        """+        axes = validate_axis_style_args(self, args, kwargs, 'mapper', 'rename')+        kwargs.update(axes)+        # Pop these, since the values are in `kwargs` under different names+        kwargs.pop('axis', None)+        kwargs.pop('mapper', None)+        return super(DataFrame, self).rename(**kwargs)++    @Substitution(**_shared_doc_kwargs)+    @Appender(NDFrame.fillna.__doc__)+    def fillna(self, value=None, method=None, axis=None, inplace=False,+               limit=None, downcast=None, **kwargs):+        return super(DataFrame,+                     self).fillna(value=value, method=method, axis=axis,+                                  inplace=inplace, limit=limit,+                                  downcast=downcast, **kwargs)++    @Appender(_shared_docs['replace'] % _shared_doc_kwargs)+    def replace(self, to_replace=None, value=None, inplace=False, limit=None,+                regex=False, method='pad'):+        return super(DataFrame, self).replace(to_replace=to_replace,+                                              value=value, inplace=inplace,+                                              limit=limit, regex=regex,+                                              method=method)++    @Appender(_shared_docs['shift'] % _shared_doc_kwargs)+    def shift(self, periods=1, freq=None, axis=0):+        return super(DataFrame, self).shift(periods=periods, freq=freq,+                                            axis=axis)++    def set_index(self, keys, drop=True, append=False, inplace=False,+                  verify_integrity=False):+        """+        Set the DataFrame index (row labels) using one or more existing+        columns. By default yields a new object.++        Parameters+        ----------+        keys : column label or list of column labels / arrays+        drop : boolean, default True+            Delete columns to be used as the new index+        append : boolean, default False+            Whether to append columns to existing index+        inplace : boolean, default False+            Modify the DataFrame in place (do not create a new object)+        verify_integrity : boolean, default False+            Check the new index for duplicates. Otherwise defer the check until+            necessary. Setting to False will improve the performance of this+            method++        Examples+        --------+        >>> df = pd.DataFrame({'month': [1, 4, 7, 10],+        ...                    'year': [2012, 2014, 2013, 2014],+        ...                    'sale':[55, 40, 84, 31]})+           month  sale  year+        0  1      55    2012+        1  4      40    2014+        2  7      84    2013+        3  10     31    2014++        Set the index to become the 'month' column:++        >>> df.set_index('month')+               sale  year+        month+        1      55    2012+        4      40    2014+        7      84    2013+        10     31    2014++        Create a multi-index using columns 'year' and 'month':++        >>> df.set_index(['year', 'month'])+                    sale+        year  month+        2012  1     55+        2014  4     40+        2013  7     84+        2014  10    31++        Create a multi-index using a set of values and a column:++        >>> df.set_index([[1, 2, 3, 4], 'year'])+                 month  sale+           year+        1  2012  1      55+        2  2014  4      40+        3  2013  7      84+        4  2014  10     31++        Returns+        -------+        dataframe : DataFrame+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        if not isinstance(keys, list):+            keys = [keys]++        if inplace:+            frame = self+        else:+            frame = self.copy()++        arrays = []+        names = []+        if append:+            names = [x for x in self.index.names]+            if isinstance(self.index, MultiIndex):+                for i in range(self.index.nlevels):+                    arrays.append(self.index._get_level_values(i))+            else:+                arrays.append(self.index)++        to_remove = []+        for col in keys:+            if isinstance(col, MultiIndex):+                # append all but the last column so we don't have to modify+                # the end of this loop+                for n in range(col.nlevels - 1):+                    arrays.append(col._get_level_values(n))++                level = col._get_level_values(col.nlevels - 1)+                names.extend(col.names)+            elif isinstance(col, Series):+                level = col._values+                names.append(col.name)+            elif isinstance(col, Index):+                level = col+                names.append(col.name)+            elif isinstance(col, (list, np.ndarray, Index)):+                level = col+                names.append(None)+            else:+                level = frame[col]._values+                names.append(col)+                if drop:+                    to_remove.append(col)+            arrays.append(level)++        index = ensure_index_from_sequences(arrays, names)++        if verify_integrity and not index.is_unique:+            duplicates = index[index.duplicated()].unique()+            raise ValueError('Index has duplicate keys: {dup}'.format(+                dup=duplicates))++        for c in to_remove:+            del frame[c]++        # clear up memory usage+        index._cleanup()++        frame.index = index++        if not inplace:+            return frame++    def reset_index(self, level=None, drop=False, inplace=False, col_level=0,+                    col_fill=''):+        """+        For DataFrame with multi-level index, return new DataFrame with+        labeling information in the columns under the index names, defaulting+        to 'level_0', 'level_1', etc. if any are None. For a standard index,+        the index name will be used (if set), otherwise a default 'index' or+        'level_0' (if 'index' is already taken) will be used.++        Parameters+        ----------+        level : int, str, tuple, or list, default None+            Only remove the given levels from the index. Removes all levels by+            default+        drop : boolean, default False+            Do not try to insert index into dataframe columns. This resets+            the index to the default integer index.+        inplace : boolean, default False+            Modify the DataFrame in place (do not create a new object)+        col_level : int or str, default 0+            If the columns have multiple levels, determines which level the+            labels are inserted into. By default it is inserted into the first+            level.+        col_fill : object, default ''+            If the columns have multiple levels, determines how the other+            levels are named. If None then the index name is repeated.++        Returns+        -------+        resetted : DataFrame++        Examples+        --------+        >>> df = pd.DataFrame([('bird',    389.0),+        ...                    ('bird',     24.0),+        ...                    ('mammal',   80.5),+        ...                    ('mammal', np.nan)],+        ...                   index=['falcon', 'parrot', 'lion', 'monkey'],+        ...                   columns=('class', 'max_speed'))+        >>> df+                 class  max_speed+        falcon    bird      389.0+        parrot    bird       24.0+        lion    mammal       80.5+        monkey  mammal        NaN++        When we reset the index, the old index is added as a column, and a+        new sequential index is used:++        >>> df.reset_index()+            index   class  max_speed+        0  falcon    bird      389.0+        1  parrot    bird       24.0+        2    lion  mammal       80.5+        3  monkey  mammal        NaN++        We can use the `drop` parameter to avoid the old index being added as+        a column:++        >>> df.reset_index(drop=True)+            class  max_speed+        0    bird      389.0+        1    bird       24.0+        2  mammal       80.5+        3  mammal        NaN++        You can also use `reset_index` with `MultiIndex`.++        >>> index = pd.MultiIndex.from_tuples([('bird', 'falcon'),+        ...                                    ('bird', 'parrot'),+        ...                                    ('mammal', 'lion'),+        ...                                    ('mammal', 'monkey')],+        ...                                   names=['class', 'name'])+        >>> columns = pd.MultiIndex.from_tuples([('speed', 'max'),+        ...                                      ('species', 'type')])+        >>> df = pd.DataFrame([(389.0, 'fly'),+        ...                    ( 24.0, 'fly'),+        ...                    ( 80.5, 'run'),+        ...                    (np.nan, 'jump')],+        ...                   index=index,+        ...                   columns=columns)+        >>> df+                       speed species+                         max    type+        class  name+        bird   falcon  389.0     fly+               parrot   24.0     fly+        mammal lion     80.5     run+               monkey    NaN    jump++        If the index has multiple levels, we can reset a subset of them:++        >>> df.reset_index(level='class')+                 class  speed species+                          max    type+        name+        falcon    bird  389.0     fly+        parrot    bird   24.0     fly+        lion    mammal   80.5     run+        monkey  mammal    NaN    jump++        If we are not dropping the index, by default, it is placed in the top+        level. We can place it in another level:++        >>> df.reset_index(level='class', col_level=1)+                        speed species+                 class    max    type+        name+        falcon    bird  389.0     fly+        parrot    bird   24.0     fly+        lion    mammal   80.5     run+        monkey  mammal    NaN    jump++        When the index is inserted under another level, we can specify under+        which one with the parameter `col_fill`:++        >>> df.reset_index(level='class', col_level=1, col_fill='species')+                      species  speed species+                        class    max    type+        name+        falcon           bird  389.0     fly+        parrot           bird   24.0     fly+        lion           mammal   80.5     run+        monkey         mammal    NaN    jump++        If we specify a nonexistent level for `col_fill`, it is created:++        >>> df.reset_index(level='class', col_level=1, col_fill='genus')+                        genus  speed species+                        class    max    type+        name+        falcon           bird  389.0     fly+        parrot           bird   24.0     fly+        lion           mammal   80.5     run+        monkey         mammal    NaN    jump+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        if inplace:+            new_obj = self+        else:+            new_obj = self.copy()++        def _maybe_casted_values(index, labels=None):+            values = index._values+            if not isinstance(index, (PeriodIndex, DatetimeIndex)):+                if values.dtype == np.object_:+                    values = lib.maybe_convert_objects(values)++            # if we have the labels, extract the values with a mask+            if labels is not None:+                mask = labels == -1++                # we can have situations where the whole mask is -1,+                # meaning there is nothing found in labels, so make all nan's+                if mask.all():+                    values = np.empty(len(mask))+                    values.fill(np.nan)+                else:+                    values = values.take(labels)+                    if mask.any():+                        values, changed = maybe_upcast_putmask(+                            values, mask, np.nan)+            return values++        new_index = ibase.default_index(len(new_obj))+        if level is not None:+            if not isinstance(level, (tuple, list)):+                level = [level]+            level = [self.index._get_level_number(lev) for lev in level]+            if len(level) < self.index.nlevels:+                new_index = self.index.droplevel(level)++        if not drop:+            if isinstance(self.index, MultiIndex):+                names = [n if n is not None else ('level_%d' % i)+                         for (i, n) in enumerate(self.index.names)]+                to_insert = lzip(self.index.levels, self.index.labels)+            else:+                default = 'index' if 'index' not in self else 'level_0'+                names = ([default] if self.index.name is None+                         else [self.index.name])+                to_insert = ((self.index, None),)++            multi_col = isinstance(self.columns, MultiIndex)+            for i, (lev, lab) in reversed(list(enumerate(to_insert))):+                if not (level is None or i in level):+                    continue+                name = names[i]+                if multi_col:+                    col_name = (list(name) if isinstance(name, tuple)+                                else [name])+                    if col_fill is None:+                        if len(col_name) not in (1, self.columns.nlevels):+                            raise ValueError("col_fill=None is incompatible "+                                             "with incomplete column name "+                                             "{}".format(name))+                        col_fill = col_name[0]++                    lev_num = self.columns._get_level_number(col_level)+                    name_lst = [col_fill] * lev_num + col_name+                    missing = self.columns.nlevels - len(name_lst)+                    name_lst += [col_fill] * missing+                    name = tuple(name_lst)+                # to ndarray and maybe infer different dtype+                level_values = _maybe_casted_values(lev, lab)+                new_obj.insert(0, name, level_values)++        new_obj.index = new_index+        if not inplace:+            return new_obj++    # ----------------------------------------------------------------------+    # Reindex-based selection methods++    @Appender(_shared_docs['isna'] % _shared_doc_kwargs)+    def isna(self):+        return super(DataFrame, self).isna()++    @Appender(_shared_docs['isna'] % _shared_doc_kwargs)+    def isnull(self):+        return super(DataFrame, self).isnull()++    @Appender(_shared_docs['notna'] % _shared_doc_kwargs)+    def notna(self):+        return super(DataFrame, self).notna()++    @Appender(_shared_docs['notna'] % _shared_doc_kwargs)+    def notnull(self):+        return super(DataFrame, self).notnull()++    def dropna(self, axis=0, how='any', thresh=None, subset=None,+               inplace=False):+        """+        Remove missing values.++        See the :ref:`User Guide <missing_data>` for more on which values are+        considered missing, and how to work with missing data.++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns'}, default 0+            Determine if rows or columns which contain missing values are+            removed.++            * 0, or 'index' : Drop rows which contain missing values.+            * 1, or 'columns' : Drop columns which contain missing value.++            .. deprecated:: 0.23.0++               Pass tuple or list to drop on multiple axes.+               Only a single axis is allowed.++        how : {'any', 'all'}, default 'any'+            Determine if row or column is removed from DataFrame, when we have+            at least one NA or all NA.++            * 'any' : If any NA values are present, drop that row or column.+            * 'all' : If all values are NA, drop that row or column.++        thresh : int, optional+            Require that many non-NA values.+        subset : array-like, optional+            Labels along other axis to consider, e.g. if you are dropping rows+            these would be a list of columns to include.+        inplace : bool, default False+            If True, do operation inplace and return None.++        Returns+        -------+        DataFrame+            DataFrame with NA entries dropped from it.++        See Also+        --------+        DataFrame.isna: Indicate missing values.+        DataFrame.notna : Indicate existing (non-missing) values.+        DataFrame.fillna : Replace missing values.+        Series.dropna : Drop missing values.+        Index.dropna : Drop missing indices.++        Examples+        --------+        >>> df = pd.DataFrame({"name": ['Alfred', 'Batman', 'Catwoman'],+        ...                    "toy": [np.nan, 'Batmobile', 'Bullwhip'],+        ...                    "born": [pd.NaT, pd.Timestamp("1940-04-25"),+        ...                             pd.NaT]})+        >>> df+               name        toy       born+        0    Alfred        NaN        NaT+        1    Batman  Batmobile 1940-04-25+        2  Catwoman   Bullwhip        NaT++        Drop the rows where at least one element is missing.++        >>> df.dropna()+             name        toy       born+        1  Batman  Batmobile 1940-04-25++        Drop the columns where at least one element is missing.++        >>> df.dropna(axis='columns')+               name+        0    Alfred+        1    Batman+        2  Catwoman++        Drop the rows where all elements are missing.++        >>> df.dropna(how='all')+               name        toy       born+        0    Alfred        NaN        NaT+        1    Batman  Batmobile 1940-04-25+        2  Catwoman   Bullwhip        NaT++        Keep only the rows with at least 2 non-NA values.++        >>> df.dropna(thresh=2)+               name        toy       born+        1    Batman  Batmobile 1940-04-25+        2  Catwoman   Bullwhip        NaT++        Define in which columns to look for missing values.++        >>> df.dropna(subset=['name', 'born'])+               name        toy       born+        1    Batman  Batmobile 1940-04-25++        Keep the DataFrame with valid entries in the same variable.++        >>> df.dropna(inplace=True)+        >>> df+             name        toy       born+        1  Batman  Batmobile 1940-04-25+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        if isinstance(axis, (tuple, list)):+            # GH20987+            msg = ("supplying multiple axes to axis is deprecated and "+                   "will be removed in a future version.")+            warnings.warn(msg, FutureWarning, stacklevel=2)++            result = self+            for ax in axis:+                result = result.dropna(how=how, thresh=thresh, subset=subset,+                                       axis=ax)+        else:+            axis = self._get_axis_number(axis)+            agg_axis = 1 - axis++            agg_obj = self+            if subset is not None:+                ax = self._get_axis(agg_axis)+                indices = ax.get_indexer_for(subset)+                check = indices == -1+                if check.any():+                    raise KeyError(list(np.compress(check, subset)))+                agg_obj = self.take(indices, axis=agg_axis)++            count = agg_obj.count(axis=agg_axis)++            if thresh is not None:+                mask = count >= thresh+            elif how == 'any':+                mask = count == len(agg_obj._get_axis(agg_axis))+            elif how == 'all':+                mask = count > 0+            else:+                if how is not None:+                    raise ValueError('invalid how option: {h}'.format(h=how))+                else:+                    raise TypeError('must specify how or thresh')++            result = self._take(mask.nonzero()[0], axis=axis)++        if inplace:+            self._update_inplace(result)+        else:+            return result++    def drop_duplicates(self, subset=None, keep='first', inplace=False):+        """+        Return DataFrame with duplicate rows removed, optionally only+        considering certain columns++        Parameters+        ----------+        subset : column label or sequence of labels, optional+            Only consider certain columns for identifying duplicates, by+            default use all of the columns+        keep : {'first', 'last', False}, default 'first'+            - ``first`` : Drop duplicates except for the first occurrence.+            - ``last`` : Drop duplicates except for the last occurrence.+            - False : Drop all duplicates.+        inplace : boolean, default False+            Whether to drop duplicates in place or to return a copy++        Returns+        -------+        deduplicated : DataFrame+        """+        inplace = validate_bool_kwarg(inplace, 'inplace')+        duplicated = self.duplicated(subset, keep=keep)++        if inplace:+            inds, = (-duplicated).nonzero()+            new_data = self._data.take(inds)+            self._update_inplace(new_data)+        else:+            return self[-duplicated]++    def duplicated(self, subset=None, keep='first'):+        """+        Return boolean Series denoting duplicate rows, optionally only+        considering certain columns++        Parameters+        ----------+        subset : column label or sequence of labels, optional+            Only consider certain columns for identifying duplicates, by+            default use all of the columns+        keep : {'first', 'last', False}, default 'first'+            - ``first`` : Mark duplicates as ``True`` except for the+              first occurrence.+            - ``last`` : Mark duplicates as ``True`` except for the+              last occurrence.+            - False : Mark all duplicates as ``True``.++        Returns+        -------+        duplicated : Series+        """+        from pandas.core.sorting import get_group_index+        from pandas._libs.hashtable import duplicated_int64, _SIZE_HINT_LIMIT++        def f(vals):+            labels, shape = algorithms.factorize(+                vals, size_hint=min(len(self), _SIZE_HINT_LIMIT))+            return labels.astype('i8', copy=False), len(shape)++        if subset is None:+            subset = self.columns+        elif (not np.iterable(subset) or+              isinstance(subset, compat.string_types) or+              isinstance(subset, tuple) and subset in self.columns):+            subset = subset,++        # Verify all columns in subset exist in the queried dataframe+        # Otherwise, raise a KeyError, same as if you try to __getitem__ with a+        # key that doesn't exist.+        diff = Index(subset).difference(self.columns)+        if not diff.empty:+            raise KeyError(diff)++        vals = (col.values for name, col in self.iteritems()+                if name in subset)+        labels, shape = map(list, zip(*map(f, vals)))++        ids = get_group_index(labels, shape, sort=False, xnull=False)+        return Series(duplicated_int64(ids, keep), index=self.index)++    # ----------------------------------------------------------------------+    # Sorting++    @Appender(_shared_docs['sort_values'] % _shared_doc_kwargs)+    def sort_values(self, by, axis=0, ascending=True, inplace=False,+                    kind='quicksort', na_position='last'):+        inplace = validate_bool_kwarg(inplace, 'inplace')+        axis = self._get_axis_number(axis)+        stacklevel = 2  # Number of stack levels from df.sort_values++        if not isinstance(by, list):+            by = [by]+        if is_sequence(ascending) and len(by) != len(ascending):+            raise ValueError('Length of ascending (%d) != length of by (%d)' %+                             (len(ascending), len(by)))+        if len(by) > 1:+            from pandas.core.sorting import lexsort_indexer++            keys = []+            for x in by:+                k = self._get_label_or_level_values(x, axis=axis,+                                                    stacklevel=stacklevel)+                keys.append(k)+            indexer = lexsort_indexer(keys, orders=ascending,+                                      na_position=na_position)+            indexer = ensure_platform_int(indexer)+        else:+            from pandas.core.sorting import nargsort++            by = by[0]+            k = self._get_label_or_level_values(by, axis=axis,+                                                stacklevel=stacklevel)++            if isinstance(ascending, (tuple, list)):+                ascending = ascending[0]++            indexer = nargsort(k, kind=kind, ascending=ascending,+                               na_position=na_position)++        new_data = self._data.take(indexer,+                                   axis=self._get_block_manager_axis(axis),+                                   verify=False)++        if inplace:+            return self._update_inplace(new_data)+        else:+            return self._constructor(new_data).__finalize__(self)++    @Appender(_shared_docs['sort_index'] % _shared_doc_kwargs)+    def sort_index(self, axis=0, level=None, ascending=True, inplace=False,+                   kind='quicksort', na_position='last', sort_remaining=True,+                   by=None):++        # TODO: this can be combined with Series.sort_index impl as+        # almost identical++        inplace = validate_bool_kwarg(inplace, 'inplace')+        # 10726+        if by is not None:+            warnings.warn("by argument to sort_index is deprecated, "+                          "please use .sort_values(by=...)",+                          FutureWarning, stacklevel=2)+            if level is not None:+                raise ValueError("unable to simultaneously sort by and level")+            return self.sort_values(by, axis=axis, ascending=ascending,+                                    inplace=inplace)++        axis = self._get_axis_number(axis)+        labels = self._get_axis(axis)++        # make sure that the axis is lexsorted to start+        # if not we need to reconstruct to get the correct indexer+        labels = labels._sort_levels_monotonic()+        if level is not None:++            new_axis, indexer = labels.sortlevel(level, ascending=ascending,+                                                 sort_remaining=sort_remaining)++        elif isinstance(labels, MultiIndex):+            from pandas.core.sorting import lexsort_indexer++            indexer = lexsort_indexer(labels._get_labels_for_sorting(),+                                      orders=ascending,+                                      na_position=na_position)+        else:+            from pandas.core.sorting import nargsort++            # Check monotonic-ness before sort an index+            # GH11080+            if ((ascending and labels.is_monotonic_increasing) or+                    (not ascending and labels.is_monotonic_decreasing)):+                if inplace:+                    return+                else:+                    return self.copy()++            indexer = nargsort(labels, kind=kind, ascending=ascending,+                               na_position=na_position)++        baxis = self._get_block_manager_axis(axis)+        new_data = self._data.take(indexer,+                                   axis=baxis,+                                   verify=False)++        # reconstruct axis if needed+        new_data.axes[baxis] = new_data.axes[baxis]._sort_levels_monotonic()++        if inplace:+            return self._update_inplace(new_data)+        else:+            return self._constructor(new_data).__finalize__(self)++    def sortlevel(self, level=0, axis=0, ascending=True, inplace=False,+                  sort_remaining=True):+        """Sort multilevel index by chosen axis and primary level. Data will be+        lexicographically sorted by the chosen level followed by the other+        levels (in order).++        .. deprecated:: 0.20.0+            Use :meth:`DataFrame.sort_index`+++        Parameters+        ----------+        level : int+        axis : {0 or 'index', 1 or 'columns'}, default 0+        ascending : boolean, default True+        inplace : boolean, default False+            Sort the DataFrame without creating a new instance+        sort_remaining : boolean, default True+            Sort by the other levels too.++        Returns+        -------+        sorted : DataFrame++        See Also+        --------+        DataFrame.sort_index(level=...)++        """+        warnings.warn("sortlevel is deprecated, use sort_index(level= ...)",+                      FutureWarning, stacklevel=2)+        return self.sort_index(level=level, axis=axis, ascending=ascending,+                               inplace=inplace, sort_remaining=sort_remaining)++    def nlargest(self, n, columns, keep='first'):+        """+        Return the first `n` rows ordered by `columns` in descending order.++        Return the first `n` rows with the largest values in `columns`, in+        descending order. The columns that are not specified are returned as+        well, but not used for ordering.++        This method is equivalent to+        ``df.sort_values(columns, ascending=False).head(n)``, but more+        performant.++        Parameters+        ----------+        n : int+            Number of rows to return.+        columns : label or list of labels+            Column label(s) to order by.+        keep : {'first', 'last', 'all'}, default 'first'+            Where there are duplicate values:++            - `first` : prioritize the first occurrence(s)+            - `last` : prioritize the last occurrence(s)+            - ``all`` : do not drop any duplicates, even it means+                        selecting more than `n` items.++            .. versionadded:: 0.24.0++        Returns+        -------+        DataFrame+            The first `n` rows ordered by the given columns in descending+            order.++        See Also+        --------+        DataFrame.nsmallest : Return the first `n` rows ordered by `columns` in+            ascending order.+        DataFrame.sort_values : Sort DataFrame by the values+        DataFrame.head : Return the first `n` rows without re-ordering.++        Notes+        -----+        This function cannot be used with all column types. For example, when+        specifying columns with `object` or `category` dtypes, ``TypeError`` is+        raised.++        Examples+        --------+        >>> df = pd.DataFrame({'a': [1, 10, 8, 11, 8, 2],+        ...                    'b': list('abdcef'),+        ...                    'c': [1.0, 2.0, np.nan, 3.0, 4.0, 9.0]})+        >>> df+            a  b    c+        0   1  a  1.0+        1  10  b  2.0+        2   8  d  NaN+        3  11  c  3.0+        4   8  e  4.0+        5   2  f  9.0++        In the following example, we will use ``nlargest`` to select the three+        rows having the largest values in column "a".++        >>> df.nlargest(3, 'a')+            a  b    c+        3  11  c  3.0+        1  10  b  2.0+        2   8  d  NaN++        When using ``keep='last'``, ties are resolved in reverse order:++        >>> df.nlargest(3, 'a', keep='last')+            a  b    c+        3  11  c  3.0+        1  10  b  2.0+        4   8  e  4.0++        When using ``keep='all'``, all duplicate items are maintained:++        >>> df.nlargest(3, 'a', keep='all')+            a  b    c+        3  11  c  3.0+        1  10  b  2.0+        2   8  d  NaN+        4   8  e  4.0++        To order by the largest values in column "a" and then "c", we can+        specify multiple columns like in the next example.++        >>> df.nlargest(3, ['a', 'c'])+            a  b    c+        4   8  e  4.0+        3  11  c  3.0+        1  10  b  2.0++        Attempting to use ``nlargest`` on non-numeric dtypes will raise a+        ``TypeError``:++        >>> df.nlargest(3, 'b')++        Traceback (most recent call last):+        TypeError: Column 'b' has dtype object, cannot use method 'nlargest'+        """+        return algorithms.SelectNFrame(self,+                                       n=n,+                                       keep=keep,+                                       columns=columns).nlargest()++    def nsmallest(self, n, columns, keep='first'):+        """Get the rows of a DataFrame sorted by the `n` smallest+        values of `columns`.++        Parameters+        ----------+        n : int+            Number of items to retrieve+        columns : list or str+            Column name or names to order by+        keep : {'first', 'last', 'all'}, default 'first'+            Where there are duplicate values:++            - ``first`` : take the first occurrence.+            - ``last`` : take the last occurrence.+            - ``all`` : do not drop any duplicates, even it means+              selecting more than `n` items.++            .. versionadded:: 0.24.0++        Returns+        -------+        DataFrame++        Examples+        --------+        >>> df = pd.DataFrame({'a': [1, 10, 8, 11, 8, 2],+        ...                    'b': list('abdcef'),+        ...                    'c': [1.0, 2.0, np.nan, 3.0, 4.0, 9.0]})+        >>> df+            a  b    c+        0   1  a  1.0+        1  10  b  2.0+        2   8  d  NaN+        3  11  c  3.0+        4   8  e  4.0+        5   2  f  9.0++        In the following example, we will use ``nsmallest`` to select the+        three rows having the smallest values in column "a".++        >>> df.nsmallest(3, 'a')+           a  b    c+        0  1  a  1.0+        5  2  f  9.0+        2  8  d  NaN++        When using ``keep='last'``, ties are resolved in reverse order:++        >>> df.nsmallest(3, 'a', keep='last')+           a  b    c+        0  1  a  1.0+        5  2  f  9.0+        4  8  e  4.0++        When using ``keep='all'``, all duplicate items are maintained:++        >>> df.nsmallest(3, 'a', keep='all')+           a  b    c+        0  1  a  1.0+        5  2  f  9.0+        2  8  d  NaN+        4  8  e  4.0++        To order by the largest values in column "a" and then "c", we can+        specify multiple columns like in the next example.++        >>> df.nsmallest(3, ['a', 'c'])+           a  b    c+        0  1  a  1.0+        5  2  f  9.0+        4  8  e  4.0++        Attempting to use ``nsmallest`` on non-numeric dtypes will raise a+        ``TypeError``:++        >>> df.nsmallest(3, 'b')++        Traceback (most recent call last):+        TypeError: Column 'b' has dtype object, cannot use method 'nsmallest'+        """+        return algorithms.SelectNFrame(self,+                                       n=n,+                                       keep=keep,+                                       columns=columns).nsmallest()++    def swaplevel(self, i=-2, j=-1, axis=0):+        """+        Swap levels i and j in a MultiIndex on a particular axis++        Parameters+        ----------+        i, j : int, string (can be mixed)+            Level of index to be swapped. Can pass level name as string.++        Returns+        -------+        swapped : same type as caller (new object)++        .. versionchanged:: 0.18.1++           The indexes ``i`` and ``j`` are now optional, and default to+           the two innermost levels of the index.++        """+        result = self.copy()++        axis = self._get_axis_number(axis)+        if axis == 0:+            result.index = result.index.swaplevel(i, j)+        else:+            result.columns = result.columns.swaplevel(i, j)+        return result++    def reorder_levels(self, order, axis=0):+        """+        Rearrange index levels using input order.+        May not drop or duplicate levels++        Parameters+        ----------+        order : list of int or list of str+            List representing new level order. Reference level by number+            (position) or by key (label).+        axis : int+            Where to reorder levels.++        Returns+        -------+        type of caller (new object)+        """+        axis = self._get_axis_number(axis)+        if not isinstance(self._get_axis(axis),+                          MultiIndex):  # pragma: no cover+            raise TypeError('Can only reorder levels on a hierarchical axis.')++        result = self.copy()++        if axis == 0:+            result.index = result.index.reorder_levels(order)+        else:+            result.columns = result.columns.reorder_levels(order)+        return result++    # ----------------------------------------------------------------------+    # Arithmetic / combination related++    def _combine_frame(self, other, func, fill_value=None, level=None):+        this, other = self.align(other, join='outer', level=level, copy=False)+        new_index, new_columns = this.index, this.columns++        def _arith_op(left, right):+            # for the mixed_type case where we iterate over columns,+            # _arith_op(left, right) is equivalent to+            # left._binop(right, func, fill_value=fill_value)+            left, right = ops.fill_binop(left, right, fill_value)+            return func(left, right)++        if this._is_mixed_type or other._is_mixed_type:+            # iterate over columns+            return ops.dispatch_to_series(this, other, _arith_op)+        else:+            result = _arith_op(this.values, other.values)++        return self._constructor(result, index=new_index, columns=new_columns,+                                 copy=False)++    def _combine_match_index(self, other, func, level=None):+        left, right = self.align(other, join='outer', axis=0, level=level,+                                 copy=False)+        new_data = func(left.values.T, right.values).T+        return self._constructor(new_data,+                                 index=left.index, columns=self.columns,+                                 copy=False)++    def _combine_match_columns(self, other, func, level=None, try_cast=True):+        left, right = self.align(other, join='outer', axis=1, level=level,+                                 copy=False)++        new_data = left._data.eval(func=func, other=right,+                                   axes=[left.columns, self.index],+                                   try_cast=try_cast)+        return self._constructor(new_data)++    def _combine_const(self, other, func, errors='raise', try_cast=True):+        if lib.is_scalar(other) or np.ndim(other) == 0:+            new_data = {i: func(self.iloc[:, i], other)+                        for i, col in enumerate(self.columns)}++            result = self._constructor(new_data, index=self.index, copy=False)+            result.columns = self.columns+            return result++        new_data = self._data.eval(func=func, other=other,+                                   errors=errors,+                                   try_cast=try_cast)+        return self._constructor(new_data)++    def _compare_frame(self, other, func, str_rep):+        # compare_frame assumes self._indexed_same(other)++        import pandas.core.computation.expressions as expressions++        def _compare(a, b):+            return {i: func(a.iloc[:, i], b.iloc[:, i])+                    for i in range(len(a.columns))}++        new_data = expressions.evaluate(_compare, str_rep, self, other)+        result = self._constructor(data=new_data, index=self.index,+                                   copy=False)+        result.columns = self.columns+        return result++    def combine(self, other, func, fill_value=None, overwrite=True):+        """+        Perform column-wise combine with another DataFrame based on a+        passed function.++        Combines a DataFrame with `other` DataFrame using `func`+        to element-wise combine columns. The row and column indexes of the+        resulting DataFrame will be the union of the two.++        Parameters+        ----------+        other : DataFrame+            The DataFrame to merge column-wise.+        func : function+            Function that takes two series as inputs and return a Series or a+            scalar. Used to merge the two dataframes column by columns.+        fill_value : scalar value, default None+            The value to fill NaNs with prior to passing any column to the+            merge func.+        overwrite : boolean, default True+            If True, columns in `self` that do not exist in `other` will be+            overwritten with NaNs.++        Returns+        -------+        result : DataFrame++        Examples+        --------+        Combine using a simple function that chooses the smaller column.++        >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [4, 4]})+        >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})+        >>> take_smaller = lambda s1, s2: s1 if s1.sum() < s2.sum() else s2+        >>> df1.combine(df2, take_smaller)+           A  B+        0  0  3+        1  0  3++        Example using a true element-wise combine function.++        >>> df1 = pd.DataFrame({'A': [5, 0], 'B': [2, 4]})+        >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})+        >>> df1.combine(df2, np.minimum)+           A  B+        0  1  2+        1  0  3++        Using `fill_value` fills Nones prior to passing the column to the+        merge function.++        >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [None, 4]})+        >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})+        >>> df1.combine(df2, take_smaller, fill_value=-5)+           A    B+        0  0 -5.0+        1  0  4.0++        However, if the same element in both dataframes is None, that None+        is preserved++        >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [None, 4]})+        >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [None, 3]})+        >>> df1.combine(df2, take_smaller, fill_value=-5)+           A    B+        0  0  NaN+        1  0  3.0++        Example that demonstrates the use of `overwrite` and behavior when+        the axis differ between the dataframes.++        >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [4, 4]})+        >>> df2 = pd.DataFrame({'B': [3, 3], 'C': [-10, 1],}, index=[1, 2])+        >>> df1.combine(df2, take_smaller)+             A    B     C+        0  NaN  NaN   NaN+        1  NaN  3.0 -10.0+        2  NaN  3.0   1.0++        >>> df1.combine(df2, take_smaller, overwrite=False)+             A    B     C+        0  0.0  NaN   NaN+        1  0.0  3.0 -10.0+        2  NaN  3.0   1.0++        Demonstrating the preference of the passed in dataframe.++        >>> df2 = pd.DataFrame({'B': [3, 3], 'C': [1, 1],}, index=[1, 2])+        >>> df2.combine(df1, take_smaller)+           A    B   C+        0  0.0  NaN NaN+        1  0.0  3.0 NaN+        2  NaN  3.0 NaN++        >>> df2.combine(df1, take_smaller, overwrite=False)+             A    B   C+        0  0.0  NaN NaN+        1  0.0  3.0 1.0+        2  NaN  3.0 1.0++        See Also+        --------+        DataFrame.combine_first : Combine two DataFrame objects and default to+            non-null values in frame calling the method+        """+        other_idxlen = len(other.index)  # save for compare++        this, other = self.align(other, copy=False)+        new_index = this.index++        if other.empty and len(new_index) == len(self.index):+            return self.copy()++        if self.empty and len(other) == other_idxlen:+            return other.copy()++        # sorts if possible+        new_columns = this.columns.union(other.columns)+        do_fill = fill_value is not None+        result = {}+        for col in new_columns:+            series = this[col]+            otherSeries = other[col]++            this_dtype = series.dtype+            other_dtype = otherSeries.dtype++            this_mask = isna(series)+            other_mask = isna(otherSeries)++            # don't overwrite columns unecessarily+            # DO propagate if this column is not in the intersection+            if not overwrite and other_mask.all():+                result[col] = this[col].copy()+                continue++            if do_fill:+                series = series.copy()+                otherSeries = otherSeries.copy()+                series[this_mask] = fill_value+                otherSeries[other_mask] = fill_value++            # if we have different dtypes, possibly promote+            new_dtype = this_dtype+            if not is_dtype_equal(this_dtype, other_dtype):+                new_dtype = find_common_type([this_dtype, other_dtype])+                if not is_dtype_equal(this_dtype, new_dtype):+                    series = series.astype(new_dtype)+                if not is_dtype_equal(other_dtype, new_dtype):+                    otherSeries = otherSeries.astype(new_dtype)++            # see if we need to be represented as i8 (datetimelike)+            # try to keep us at this dtype+            needs_i8_conversion_i = needs_i8_conversion(new_dtype)+            if needs_i8_conversion_i:+                arr = func(series, otherSeries, True)+            else:+                arr = func(series, otherSeries)++            arr = maybe_downcast_to_dtype(arr, this_dtype)++            result[col] = arr++        # convert_objects just in case+        return self._constructor(result, index=new_index,+                                 columns=new_columns)._convert(datetime=True,+                                                               copy=False)++    def combine_first(self, other):+        """+        Update null elements with value in the same location in `other`.++        Combine two DataFrame objects by filling null values in one DataFrame+        with non-null values from other DataFrame. The row and column indexes+        of the resulting DataFrame will be the union of the two.++        Parameters+        ----------+        other : DataFrame+            Provided DataFrame to use to fill null values.++        Returns+        -------+        combined : DataFrame++        Examples+        --------++        >>> df1 = pd.DataFrame({'A': [None, 0], 'B': [None, 4]})+        >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})+        >>> df1.combine_first(df2)+             A    B+        0  1.0  3.0+        1  0.0  4.0++        Null values still persist if the location of that null value+        does not exist in `other`++        >>> df1 = pd.DataFrame({'A': [None, 0], 'B': [4, None]})+        >>> df2 = pd.DataFrame({'B': [3, 3], 'C': [1, 1]}, index=[1, 2])+        >>> df1.combine_first(df2)+             A    B    C+        0  NaN  4.0  NaN+        1  0.0  3.0  1.0+        2  NaN  3.0  1.0++        See Also+        --------+        DataFrame.combine : Perform series-wise operation on two DataFrames+            using a given function+        """+        import pandas.core.computation.expressions as expressions++        def combiner(x, y, needs_i8_conversion=False):+            x_values = x.values if hasattr(x, 'values') else x+            y_values = y.values if hasattr(y, 'values') else y+            if needs_i8_conversion:+                mask = isna(x)+                x_values = x_values.view('i8')+                y_values = y_values.view('i8')+            else:+                mask = isna(x_values)++            return expressions.where(mask, y_values, x_values)++        return self.combine(other, combiner, overwrite=False)++    def update(self, other, join='left', overwrite=True, filter_func=None,+               raise_conflict=False):+        """+        Modify in place using non-NA values from another DataFrame.++        Aligns on indices. There is no return value.++        Parameters+        ----------+        other : DataFrame, or object coercible into a DataFrame+            Should have at least one matching index/column label+            with the original DataFrame. If a Series is passed,+            its name attribute must be set, and that will be+            used as the column name to align with the original DataFrame.+        join : {'left'}, default 'left'+            Only left join is implemented, keeping the index and columns of the+            original object.+        overwrite : bool, default True+            How to handle non-NA values for overlapping keys:++            * True: overwrite original DataFrame's values+              with values from `other`.+            * False: only update values that are NA in+              the original DataFrame.++        filter_func : callable(1d-array) -> boolean 1d-array, optional+            Can choose to replace values other than NA. Return True for values+            that should be updated.+        raise_conflict : bool, default False+            If True, will raise a ValueError if the DataFrame and `other`+            both contain non-NA data in the same place.++        Raises+        ------+        ValueError+            When `raise_conflict` is True and there's overlapping non-NA data.++        See Also+        --------+        dict.update : Similar method for dictionaries.+        DataFrame.merge : For column(s)-on-columns(s) operations.++        Examples+        --------+        >>> df = pd.DataFrame({'A': [1, 2, 3],+        ...                    'B': [400, 500, 600]})+        >>> new_df = pd.DataFrame({'B': [4, 5, 6],+        ...                        'C': [7, 8, 9]})+        >>> df.update(new_df)+        >>> df+           A  B+        0  1  4+        1  2  5+        2  3  6++        The DataFrame's length does not increase as a result of the update,+        only values at matching index/column labels are updated.++        >>> df = pd.DataFrame({'A': ['a', 'b', 'c'],+        ...                    'B': ['x', 'y', 'z']})+        >>> new_df = pd.DataFrame({'B': ['d', 'e', 'f', 'g', 'h', 'i']})+        >>> df.update(new_df)+        >>> df+           A  B+        0  a  d+        1  b  e+        2  c  f++        For Series, it's name attribute must be set.++        >>> df = pd.DataFrame({'A': ['a', 'b', 'c'],+        ...                    'B': ['x', 'y', 'z']})+        >>> new_column = pd.Series(['d', 'e'], name='B', index=[0, 2])+        >>> df.update(new_column)+        >>> df+           A  B+        0  a  d+        1  b  y+        2  c  e+        >>> df = pd.DataFrame({'A': ['a', 'b', 'c'],+        ...                    'B': ['x', 'y', 'z']})+        >>> new_df = pd.DataFrame({'B': ['d', 'e']}, index=[1, 2])+        >>> df.update(new_df)+        >>> df+           A  B+        0  a  x+        1  b  d+        2  c  e++        If `other` contains NaNs the corresponding values are not updated+        in the original dataframe.++        >>> df = pd.DataFrame({'A': [1, 2, 3],+        ...                    'B': [400, 500, 600]})+        >>> new_df = pd.DataFrame({'B': [4, np.nan, 6]})+        >>> df.update(new_df)+        >>> df+           A      B+        0  1    4.0+        1  2  500.0+        2  3    6.0+        """+        import pandas.core.computation.expressions as expressions+        # TODO: Support other joins+        if join != 'left':  # pragma: no cover+            raise NotImplementedError("Only left join is supported")++        if not isinstance(other, DataFrame):+            other = DataFrame(other)++        other = other.reindex_like(self)++        for col in self.columns:+            this = self[col].values+            that = other[col].values+            if filter_func is not None:+                with np.errstate(all='ignore'):+                    mask = ~filter_func(this) | isna(that)+            else:+                if raise_conflict:+                    mask_this = notna(that)+                    mask_that = notna(this)+                    if any(mask_this & mask_that):+                        raise ValueError("Data overlaps.")++                if overwrite:+                    mask = isna(that)+                else:+                    mask = notna(this)++            # don't overwrite columns unecessarily+            if mask.all():+                continue++            self[col] = expressions.where(mask, this, that)++    # ----------------------------------------------------------------------+    # Data reshaping++    _shared_docs['pivot'] = """+        Return reshaped DataFrame organized by given index / column values.++        Reshape data (produce a "pivot" table) based on column values. Uses+        unique values from specified `index` / `columns` to form axes of the+        resulting DataFrame. This function does not support data+        aggregation, multiple values will result in a MultiIndex in the+        columns. See the :ref:`User Guide <reshaping>` for more on reshaping.++        Parameters+        ----------%s+        index : string or object, optional+            Column to use to make new frame's index. If None, uses+            existing index.+        columns : string or object+            Column to use to make new frame's columns.+        values : string, object or a list of the previous, optional+            Column(s) to use for populating new frame's values. If not+            specified, all remaining columns will be used and the result will+            have hierarchically indexed columns.++            .. versionchanged :: 0.23.0+               Also accept list of column names.++        Returns+        -------+        DataFrame+            Returns reshaped DataFrame.++        Raises+        ------+        ValueError:+            When there are any `index`, `columns` combinations with multiple+            values. `DataFrame.pivot_table` when you need to aggregate.++        See Also+        --------+        DataFrame.pivot_table : generalization of pivot that can handle+            duplicate values for one index/column pair.+        DataFrame.unstack : pivot based on the index values instead of a+            column.++        Notes+        -----+        For finer-tuned control, see hierarchical indexing documentation along+        with the related stack/unstack methods.++        Examples+        --------+        >>> df = pd.DataFrame({'foo': ['one', 'one', 'one', 'two', 'two',+        ...                            'two'],+        ...                    'bar': ['A', 'B', 'C', 'A', 'B', 'C'],+        ...                    'baz': [1, 2, 3, 4, 5, 6],+        ...                    'zoo': ['x', 'y', 'z', 'q', 'w', 't']})+        >>> df+            foo   bar  baz  zoo+        0   one   A    1    x+        1   one   B    2    y+        2   one   C    3    z+        3   two   A    4    q+        4   two   B    5    w+        5   two   C    6    t++        >>> df.pivot(index='foo', columns='bar', values='baz')+        bar  A   B   C+        foo+        one  1   2   3+        two  4   5   6++        >>> df.pivot(index='foo', columns='bar')['baz']+        bar  A   B   C+        foo+        one  1   2   3+        two  4   5   6++        >>> df.pivot(index='foo', columns='bar', values=['baz', 'zoo'])+              baz       zoo+        bar   A  B  C   A  B  C+        foo+        one   1  2  3   x  y  z+        two   4  5  6   q  w  t++        A ValueError is raised if there are any duplicates.++        >>> df = pd.DataFrame({"foo": ['one', 'one', 'two', 'two'],+        ...                    "bar": ['A', 'A', 'B', 'C'],+        ...                    "baz": [1, 2, 3, 4]})+        >>> df+           foo bar  baz+        0  one   A    1+        1  one   A    2+        2  two   B    3+        3  two   C    4++        Notice that the first two rows are the same for our `index`+        and `columns` arguments.++        >>> df.pivot(index='foo', columns='bar', values='baz')+        Traceback (most recent call last):+           ...+        ValueError: Index contains duplicate entries, cannot reshape+        """++    @Substitution('')+    @Appender(_shared_docs['pivot'])+    def pivot(self, index=None, columns=None, values=None):+        from pandas.core.reshape.pivot import pivot+        return pivot(self, index=index, columns=columns, values=values)++    _shared_docs['pivot_table'] = """+        Create a spreadsheet-style pivot table as a DataFrame. The levels in+        the pivot table will be stored in MultiIndex objects (hierarchical+        indexes) on the index and columns of the result DataFrame++        Parameters+        ----------%s+        values : column to aggregate, optional+        index : column, Grouper, array, or list of the previous+            If an array is passed, it must be the same length as the data. The+            list can contain any of the other types (except list).+            Keys to group by on the pivot table index.  If an array is passed,+            it is being used as the same manner as column values.+        columns : column, Grouper, array, or list of the previous+            If an array is passed, it must be the same length as the data. The+            list can contain any of the other types (except list).+            Keys to group by on the pivot table column.  If an array is passed,+            it is being used as the same manner as column values.+        aggfunc : function, list of functions, dict, default numpy.mean+            If list of functions passed, the resulting pivot table will have+            hierarchical columns whose top level are the function names+            (inferred from the function objects themselves)+            If dict is passed, the key is column to aggregate and value+            is function or list of functions+        fill_value : scalar, default None+            Value to replace missing values with+        margins : boolean, default False+            Add all row / columns (e.g. for subtotal / grand totals)+        dropna : boolean, default True+            Do not include columns whose entries are all NaN+        margins_name : string, default 'All'+            Name of the row / column that will contain the totals+            when margins is True.++        Examples+        --------+        >>> df = pd.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",+        ...                          "bar", "bar", "bar", "bar"],+        ...                    "B": ["one", "one", "one", "two", "two",+        ...                          "one", "one", "two", "two"],+        ...                    "C": ["small", "large", "large", "small",+        ...                          "small", "large", "small", "small",+        ...                          "large"],+        ...                    "D": [1, 2, 2, 3, 3, 4, 5, 6, 7]})+        >>> df+             A    B      C  D+        0  foo  one  small  1+        1  foo  one  large  2+        2  foo  one  large  2+        3  foo  two  small  3+        4  foo  two  small  3+        5  bar  one  large  4+        6  bar  one  small  5+        7  bar  two  small  6+        8  bar  two  large  7++        >>> table = pivot_table(df, values='D', index=['A', 'B'],+        ...                     columns=['C'], aggfunc=np.sum)+        >>> table+        C        large  small+        A   B+        bar one    4.0    5.0+            two    7.0    6.0+        foo one    4.0    1.0+            two    NaN    6.0++        >>> table = pivot_table(df, values='D', index=['A', 'B'],+        ...                     columns=['C'], aggfunc=np.sum)+        >>> table+        C        large  small+        A   B+        bar one    4.0    5.0+            two    7.0    6.0+        foo one    4.0    1.0+            two    NaN    6.0++        >>> table = pivot_table(df, values=['D', 'E'], index=['A', 'C'],+        ...                     aggfunc={'D': np.mean,+        ...                              'E': [min, max, np.mean]})+        >>> table+                          D   E+                       mean max median min+        A   C+        bar large  5.500000  16   14.5  13+            small  5.500000  15   14.5  14+        foo large  2.000000  10    9.5   9+            small  2.333333  12   11.0   8++        Returns+        -------+        table : DataFrame++        See also+        --------+        DataFrame.pivot : pivot without aggregation that can handle+            non-numeric data+        """++    @Substitution('')+    @Appender(_shared_docs['pivot_table'])+    def pivot_table(self, values=None, index=None, columns=None,+                    aggfunc='mean', fill_value=None, margins=False,+                    dropna=True, margins_name='All'):+        from pandas.core.reshape.pivot import pivot_table+        return pivot_table(self, values=values, index=index, columns=columns,+                           aggfunc=aggfunc, fill_value=fill_value,+                           margins=margins, dropna=dropna,+                           margins_name=margins_name)++    def stack(self, level=-1, dropna=True):+        """+        Stack the prescribed level(s) from columns to index.++        Return a reshaped DataFrame or Series having a multi-level+        index with one or more new inner-most levels compared to the current+        DataFrame. The new inner-most levels are created by pivoting the+        columns of the current dataframe:++          - if the columns have a single level, the output is a Series;+          - if the columns have multiple levels, the new index+            level(s) is (are) taken from the prescribed level(s) and+            the output is a DataFrame.++        The new index levels are sorted.++        Parameters+        ----------+        level : int, str, list, default -1+            Level(s) to stack from the column axis onto the index+            axis, defined as one index or label, or a list of indices+            or labels.+        dropna : bool, default True+            Whether to drop rows in the resulting Frame/Series with+            missing values. Stacking a column level onto the index+            axis can create combinations of index and column values+            that are missing from the original dataframe. See Examples+            section.++        Returns+        -------+        DataFrame or Series+            Stacked dataframe or series.++        See Also+        --------+        DataFrame.unstack : Unstack prescribed level(s) from index axis+             onto column axis.+        DataFrame.pivot : Reshape dataframe from long format to wide+             format.+        DataFrame.pivot_table : Create a spreadsheet-style pivot table+             as a DataFrame.++        Notes+        -----+        The function is named by analogy with a collection of books+        being re-organised from being side by side on a horizontal+        position (the columns of the dataframe) to being stacked+        vertically on top of of each other (in the index of the+        dataframe).++        Examples+        --------+        **Single level columns**++        >>> df_single_level_cols = pd.DataFrame([[0, 1], [2, 3]],+        ...                                     index=['cat', 'dog'],+        ...                                     columns=['weight', 'height'])++        Stacking a dataframe with a single level column axis returns a Series:++        >>> df_single_level_cols+             weight height+        cat       0      1+        dog       2      3+        >>> df_single_level_cols.stack()+        cat  weight    0+             height    1+        dog  weight    2+             height    3+        dtype: int64++        **Multi level columns: simple case**++        >>> multicol1 = pd.MultiIndex.from_tuples([('weight', 'kg'),+        ...                                        ('weight', 'pounds')])+        >>> df_multi_level_cols1 = pd.DataFrame([[1, 2], [2, 4]],+        ...                                     index=['cat', 'dog'],+        ...                                     columns=multicol1)++        Stacking a dataframe with a multi-level column axis:++        >>> df_multi_level_cols1+             weight+                 kg    pounds+        cat       1        2+        dog       2        4+        >>> df_multi_level_cols1.stack()+                    weight+        cat kg           1+            pounds       2+        dog kg           2+            pounds       4++        **Missing values**++        >>> multicol2 = pd.MultiIndex.from_tuples([('weight', 'kg'),+        ...                                        ('height', 'm')])+        >>> df_multi_level_cols2 = pd.DataFrame([[1.0, 2.0], [3.0, 4.0]],+        ...                                     index=['cat', 'dog'],+        ...                                     columns=multicol2)++        It is common to have missing values when stacking a dataframe+        with multi-level columns, as the stacked dataframe typically+        has more values than the original dataframe. Missing values+        are filled with NaNs:++        >>> df_multi_level_cols2+            weight height+                kg      m+        cat    1.0    2.0+        dog    3.0    4.0+        >>> df_multi_level_cols2.stack()+                height  weight+        cat kg     NaN     1.0+            m      2.0     NaN+        dog kg     NaN     3.0+            m      4.0     NaN++        **Prescribing the level(s) to be stacked**++        The first parameter controls which level or levels are stacked:++        >>> df_multi_level_cols2.stack(0)+                     kg    m+        cat height  NaN  2.0+            weight  1.0  NaN+        dog height  NaN  4.0+            weight  3.0  NaN+        >>> df_multi_level_cols2.stack([0, 1])+        cat  height  m     2.0+             weight  kg    1.0+        dog  height  m     4.0+             weight  kg    3.0+        dtype: float64++        **Dropping missing values**++        >>> df_multi_level_cols3 = pd.DataFrame([[None, 1.0], [2.0, 3.0]],+        ...                                     index=['cat', 'dog'],+        ...                                     columns=multicol2)++        Note that rows where all values are missing are dropped by+        default but this behaviour can be controlled via the dropna+        keyword parameter:++        >>> df_multi_level_cols3+            weight height+                kg      m+        cat    NaN    1.0+        dog    2.0    3.0+        >>> df_multi_level_cols3.stack(dropna=False)+                height  weight+        cat kg     NaN     NaN+            m      1.0     NaN+        dog kg     NaN     2.0+            m      3.0     NaN+        >>> df_multi_level_cols3.stack(dropna=True)+                height  weight+        cat m      1.0     NaN+        dog kg     NaN     2.0+            m      3.0     NaN+        """+        from pandas.core.reshape.reshape import stack, stack_multiple++        if isinstance(level, (tuple, list)):+            return stack_multiple(self, level, dropna=dropna)+        else:+            return stack(self, level, dropna=dropna)++    def unstack(self, level=-1, fill_value=None):+        """+        Pivot a level of the (necessarily hierarchical) index labels, returning+        a DataFrame having a new level of column labels whose inner-most level+        consists of the pivoted index labels. If the index is not a MultiIndex,+        the output will be a Series (the analogue of stack when the columns are+        not a MultiIndex).+        The level involved will automatically get sorted.++        Parameters+        ----------+        level : int, string, or list of these, default -1 (last level)+            Level(s) of index to unstack, can pass level name+        fill_value : replace NaN with this value if the unstack produces+            missing values++            .. versionadded:: 0.18.0++        See also+        --------+        DataFrame.pivot : Pivot a table based on column values.+        DataFrame.stack : Pivot a level of the column labels (inverse operation+            from `unstack`).++        Examples+        --------+        >>> index = pd.MultiIndex.from_tuples([('one', 'a'), ('one', 'b'),+        ...                                    ('two', 'a'), ('two', 'b')])+        >>> s = pd.Series(np.arange(1.0, 5.0), index=index)+        >>> s+        one  a   1.0+             b   2.0+        two  a   3.0+             b   4.0+        dtype: float64++        >>> s.unstack(level=-1)+             a   b+        one  1.0  2.0+        two  3.0  4.0++        >>> s.unstack(level=0)+           one  two+        a  1.0   3.0+        b  2.0   4.0++        >>> df = s.unstack(level=0)+        >>> df.unstack()+        one  a  1.0+             b  2.0+        two  a  3.0+             b  4.0+        dtype: float64++        Returns+        -------+        unstacked : DataFrame or Series+        """+        from pandas.core.reshape.reshape import unstack+        return unstack(self, level, fill_value)++    _shared_docs['melt'] = ("""+    "Unpivots" a DataFrame from wide format to long format, optionally+    leaving identifier variables set.++    This function is useful to massage a DataFrame into a format where one+    or more columns are identifier variables (`id_vars`), while all other+    columns, considered measured variables (`value_vars`), are "unpivoted" to+    the row axis, leaving just two non-identifier columns, 'variable' and+    'value'.++    %(versionadded)s+    Parameters+    ----------+    frame : DataFrame+    id_vars : tuple, list, or ndarray, optional+        Column(s) to use as identifier variables.+    value_vars : tuple, list, or ndarray, optional+        Column(s) to unpivot. If not specified, uses all columns that+        are not set as `id_vars`.+    var_name : scalar+        Name to use for the 'variable' column. If None it uses+        ``frame.columns.name`` or 'variable'.+    value_name : scalar, default 'value'+        Name to use for the 'value' column.+    col_level : int or string, optional+        If columns are a MultiIndex then use this level to melt.++    See also+    --------+    %(other)s+    pivot_table+    DataFrame.pivot++    Examples+    --------+    >>> df = pd.DataFrame({'A': {0: 'a', 1: 'b', 2: 'c'},+    ...                    'B': {0: 1, 1: 3, 2: 5},+    ...                    'C': {0: 2, 1: 4, 2: 6}})+    >>> df+       A  B  C+    0  a  1  2+    1  b  3  4+    2  c  5  6++    >>> %(caller)sid_vars=['A'], value_vars=['B'])+       A variable  value+    0  a        B      1+    1  b        B      3+    2  c        B      5++    >>> %(caller)sid_vars=['A'], value_vars=['B', 'C'])+       A variable  value+    0  a        B      1+    1  b        B      3+    2  c        B      5+    3  a        C      2+    4  b        C      4+    5  c        C      6++    The names of 'variable' and 'value' columns can be customized:++    >>> %(caller)sid_vars=['A'], value_vars=['B'],+    ...         var_name='myVarname', value_name='myValname')+       A myVarname  myValname+    0  a         B          1+    1  b         B          3+    2  c         B          5++    If you have multi-index columns:++    >>> df.columns = [list('ABC'), list('DEF')]+    >>> df+       A  B  C+       D  E  F+    0  a  1  2+    1  b  3  4+    2  c  5  6++    >>> %(caller)scol_level=0, id_vars=['A'], value_vars=['B'])+       A variable  value+    0  a        B      1+    1  b        B      3+    2  c        B      5++    >>> %(caller)sid_vars=[('A', 'D')], value_vars=[('B', 'E')])+      (A, D) variable_0 variable_1  value+    0      a          B          E      1+    1      b          B          E      3+    2      c          B          E      5++    """)++    @Appender(_shared_docs['melt'] %+              dict(caller='df.melt(',+                   versionadded='.. versionadded:: 0.20.0\n',+                   other='melt'))+    def melt(self, id_vars=None, value_vars=None, var_name=None,+             value_name='value', col_level=None):+        from pandas.core.reshape.melt import melt+        return melt(self, id_vars=id_vars, value_vars=value_vars,+                    var_name=var_name, value_name=value_name,+                    col_level=col_level)++    # ----------------------------------------------------------------------+    # Time series-related++    def diff(self, periods=1, axis=0):+        """+        First discrete difference of element.++        Calculates the difference of a DataFrame element compared with another+        element in the DataFrame (default is the element in the same column+        of the previous row).++        Parameters+        ----------+        periods : int, default 1+            Periods to shift for calculating difference, accepts negative+            values.+        axis : {0 or 'index', 1 or 'columns'}, default 0+            Take difference over rows (0) or columns (1).++            .. versionadded:: 0.16.1.++        Returns+        -------+        diffed : DataFrame++        See Also+        --------+        Series.diff: First discrete difference for a Series.+        DataFrame.pct_change: Percent change over given number of periods.+        DataFrame.shift: Shift index by desired number of periods with an+            optional time freq.++        Examples+        --------+        Difference with previous row++        >>> df = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],+        ...                    'b': [1, 1, 2, 3, 5, 8],+        ...                    'c': [1, 4, 9, 16, 25, 36]})+        >>> df+           a  b   c+        0  1  1   1+        1  2  1   4+        2  3  2   9+        3  4  3  16+        4  5  5  25+        5  6  8  36++        >>> df.diff()+             a    b     c+        0  NaN  NaN   NaN+        1  1.0  0.0   3.0+        2  1.0  1.0   5.0+        3  1.0  1.0   7.0+        4  1.0  2.0   9.0+        5  1.0  3.0  11.0++        Difference with previous column++        >>> df.diff(axis=1)+            a    b     c+        0 NaN  0.0   0.0+        1 NaN -1.0   3.0+        2 NaN -1.0   7.0+        3 NaN -1.0  13.0+        4 NaN  0.0  20.0+        5 NaN  2.0  28.0++        Difference with 3rd previous row++        >>> df.diff(periods=3)+             a    b     c+        0  NaN  NaN   NaN+        1  NaN  NaN   NaN+        2  NaN  NaN   NaN+        3  3.0  2.0  15.0+        4  3.0  4.0  21.0+        5  3.0  6.0  27.0++        Difference with following row++        >>> df.diff(periods=-1)+             a    b     c+        0 -1.0  0.0  -3.0+        1 -1.0 -1.0  -5.0+        2 -1.0 -1.0  -7.0+        3 -1.0 -2.0  -9.0+        4 -1.0 -3.0 -11.0+        5  NaN  NaN   NaN+        """+        bm_axis = self._get_block_manager_axis(axis)+        new_data = self._data.diff(n=periods, axis=bm_axis)+        return self._constructor(new_data)++    # ----------------------------------------------------------------------+    # Function application++    def _gotitem(self,+                 key,           # type: Union[str, List[str]]+                 ndim,          # type: int+                 subset=None    # type: Union[Series, DataFrame, None]+                 ):+        # type: (...) -> Union[Series, DataFrame]+        """+        sub-classes to define+        return a sliced object++        Parameters+        ----------+        key : string / list of selections+        ndim : 1,2+            requested ndim of result+        subset : object, default None+            subset to act on+        """+        if subset is None:+            subset = self+        elif subset.ndim == 1:  # is Series+            return subset++        # TODO: _shallow_copy(subset)?+        return subset[key]++    _agg_doc = dedent("""+    The aggregation operations are always performed over an axis, either the+    index (default) or the column axis. This behavior is different from+    `numpy` aggregation functions (`mean`, `median`, `prod`, `sum`, `std`,+    `var`), where the default is to compute the aggregation of the flattened+    array, e.g., ``numpy.mean(arr_2d)`` as opposed to ``numpy.mean(arr_2d,+    axis=0)``.++    `agg` is an alias for `aggregate`. Use the alias.++    Examples+    --------+    >>> df = pd.DataFrame([[1, 2, 3],+    ...                    [4, 5, 6],+    ...                    [7, 8, 9],+    ...                    [np.nan, np.nan, np.nan]],+    ...                   columns=['A', 'B', 'C'])++    Aggregate these functions over the rows.++    >>> df.agg(['sum', 'min'])+            A     B     C+    sum  12.0  15.0  18.0+    min   1.0   2.0   3.0++    Different aggregations per column.++    >>> df.agg({'A' : ['sum', 'min'], 'B' : ['min', 'max']})+            A    B+    max   NaN  8.0+    min   1.0  2.0+    sum  12.0  NaN++    Aggregate over the columns.++    >>> df.agg("mean", axis="columns")+    0    2.0+    1    5.0+    2    8.0+    3    NaN+    dtype: float64++    See also+    --------+    DataFrame.apply : Perform any type of operations.+    DataFrame.transform : Perform transformation type operations.+    pandas.core.groupby.GroupBy : Perform operations over groups.+    pandas.core.resample.Resampler : Perform operations over resampled bins.+    pandas.core.window.Rolling : Perform operations over rolling window.+    pandas.core.window.Expanding : Perform operations over expanding window.+    pandas.core.window.EWM : Perform operation over exponential weighted+        window.+    """)++    @Appender(_agg_doc)+    @Appender(_shared_docs['aggregate'] % dict(+        versionadded='.. versionadded:: 0.20.0',+        **_shared_doc_kwargs))+    def aggregate(self, func, axis=0, *args, **kwargs):+        axis = self._get_axis_number(axis)++        result = None+        try:+            result, how = self._aggregate(func, axis=axis, *args, **kwargs)+        except TypeError:+            pass+        if result is None:+            return self.apply(func, axis=axis, args=args, **kwargs)+        return result++    def _aggregate(self, arg, axis=0, *args, **kwargs):+        if axis == 1:+            # NDFrame.aggregate returns a tuple, and we need to transpose+            # only result+            result, how = (super(DataFrame, self.T)+                           ._aggregate(arg, *args, **kwargs))+            result = result.T if result is not None else result+            return result, how+        return super(DataFrame, self)._aggregate(arg, *args, **kwargs)++    agg = aggregate++    @Appender(_shared_docs['transform'] % _shared_doc_kwargs)+    def transform(self, func, axis=0, *args, **kwargs):+        axis = self._get_axis_number(axis)+        if axis == 1:+            return super(DataFrame, self.T).transform(func, *args, **kwargs).T+        return super(DataFrame, self).transform(func, *args, **kwargs)++    def apply(self, func, axis=0, broadcast=None, raw=False, reduce=None,+              result_type=None, args=(), **kwds):+        """+        Apply a function along an axis of the DataFrame.++        Objects passed to the function are Series objects whose index is+        either the DataFrame's index (``axis=0``) or the DataFrame's columns+        (``axis=1``). By default (``result_type=None``), the final return type+        is inferred from the return type of the applied function. Otherwise,+        it depends on the `result_type` argument.++        Parameters+        ----------+        func : function+            Function to apply to each column or row.+        axis : {0 or 'index', 1 or 'columns'}, default 0+            Axis along which the function is applied:++            * 0 or 'index': apply function to each column.+            * 1 or 'columns': apply function to each row.+        broadcast : bool, optional+            Only relevant for aggregation functions:++            * ``False`` or ``None`` : returns a Series whose length is the+              length of the index or the number of columns (based on the+              `axis` parameter)+            * ``True`` : results will be broadcast to the original shape+              of the frame, the original index and columns will be retained.++            .. deprecated:: 0.23.0+               This argument will be removed in a future version, replaced+               by result_type='broadcast'.++        raw : bool, default False+            * ``False`` : passes each row or column as a Series to the+              function.+            * ``True`` : the passed function will receive ndarray objects+              instead.+              If you are just applying a NumPy reduction function this will+              achieve much better performance.+        reduce : bool or None, default None+            Try to apply reduction procedures. If the DataFrame is empty,+            `apply` will use `reduce` to determine whether the result+            should be a Series or a DataFrame. If ``reduce=None`` (the+            default), `apply`'s return value will be guessed by calling+            `func` on an empty Series+            (note: while guessing, exceptions raised by `func` will be+            ignored).+            If ``reduce=True`` a Series will always be returned, and if+            ``reduce=False`` a DataFrame will always be returned.++            .. deprecated:: 0.23.0+               This argument will be removed in a future version, replaced+               by ``result_type='reduce'``.++        result_type : {'expand', 'reduce', 'broadcast', None}, default None+            These only act when ``axis=1`` (columns):++            * 'expand' : list-like results will be turned into columns.+            * 'reduce' : returns a Series if possible rather than expanding+              list-like results. This is the opposite of 'expand'.+            * 'broadcast' : results will be broadcast to the original shape+              of the DataFrame, the original index and columns will be+              retained.++            The default behaviour (None) depends on the return value of the+            applied function: list-like results will be returned as a Series+            of those. However if the apply function returns a Series these+            are expanded to columns.++            .. versionadded:: 0.23.0++        args : tuple+            Positional arguments to pass to `func` in addition to the+            array/series.+        **kwds+            Additional keyword arguments to pass as keywords arguments to+            `func`.++        Notes+        -----+        In the current implementation apply calls `func` twice on the+        first column/row to decide whether it can take a fast or slow+        code path. This can lead to unexpected behavior if `func` has+        side-effects, as they will take effect twice for the first+        column/row.++        See also+        --------+        DataFrame.applymap: For elementwise operations+        DataFrame.aggregate: only perform aggregating type operations+        DataFrame.transform: only perform transforming type operations++        Examples+        --------++        >>> df = pd.DataFrame([[4, 9],] * 3, columns=['A', 'B'])+        >>> df+           A  B+        0  4  9+        1  4  9+        2  4  9++        Using a numpy universal function (in this case the same as+        ``np.sqrt(df)``):++        >>> df.apply(np.sqrt)+             A    B+        0  2.0  3.0+        1  2.0  3.0+        2  2.0  3.0++        Using a reducing function on either axis++        >>> df.apply(np.sum, axis=0)+        A    12+        B    27+        dtype: int64++        >>> df.apply(np.sum, axis=1)+        0    13+        1    13+        2    13+        dtype: int64++        Retuning a list-like will result in a Series++        >>> df.apply(lambda x: [1, 2], axis=1)+        0    [1, 2]+        1    [1, 2]+        2    [1, 2]+        dtype: object++        Passing result_type='expand' will expand list-like results+        to columns of a Dataframe++        >>> df.apply(lambda x: [1, 2], axis=1, result_type='expand')+           0  1+        0  1  2+        1  1  2+        2  1  2++        Returning a Series inside the function is similar to passing+        ``result_type='expand'``. The resulting column names+        will be the Series index.++        >>> df.apply(lambda x: pd.Series([1, 2], index=['foo', 'bar']), axis=1)+           foo  bar+        0    1    2+        1    1    2+        2    1    2++        Passing ``result_type='broadcast'`` will ensure the same shape+        result, whether list-like or scalar is returned by the function,+        and broadcast it along the axis. The resulting column names will+        be the originals.++        >>> df.apply(lambda x: [1, 2], axis=1, result_type='broadcast')+           A  B+        0  1  2+        1  1  2+        2  1  2++        Returns+        -------+        applied : Series or DataFrame+        """+        from pandas.core.apply import frame_apply+        op = frame_apply(self,+                         func=func,+                         axis=axis,+                         broadcast=broadcast,+                         raw=raw,+                         reduce=reduce,+                         result_type=result_type,+                         args=args,+                         kwds=kwds)+        return op.get_result()++    def applymap(self, func):+        """+        Apply a function to a Dataframe elementwise.++        This method applies a function that accepts and returns a scalar+        to every element of a DataFrame.++        Parameters+        ----------+        func : callable+            Python function, returns a single value from a single value.++        Returns+        -------+        DataFrame+            Transformed DataFrame.++        See also+        --------+        DataFrame.apply : Apply a function along input axis of DataFrame++        Examples+        --------+        >>> df = pd.DataFrame([[1, 2.12], [3.356, 4.567]])+        >>> df+               0      1+        0  1.000  2.120+        1  3.356  4.567++        >>> df.applymap(lambda x: len(str(x)))+           0  1+        0  3  4+        1  5  5++        Note that a vectorized version of `func` often exists, which will+        be much faster. You could square each number elementwise.++        >>> df.applymap(lambda x: x**2)+                   0          1+        0   1.000000   4.494400+        1  11.262736  20.857489++        But it's better to avoid applymap in that case.++        >>> df ** 2+                   0          1+        0   1.000000   4.494400+        1  11.262736  20.857489+        """++        # if we have a dtype == 'M8[ns]', provide boxed values+        def infer(x):+            if x.empty:+                return lib.map_infer(x, func)+            return lib.map_infer(x.astype(object).values, func)++        return self.apply(infer)++    # ----------------------------------------------------------------------+    # Merging / joining methods++    def append(self, other, ignore_index=False,+               verify_integrity=False, sort=None):+        """+        Append rows of `other` to the end of caller, returning a new object.++        Columns in `other` that are not in the caller are added as new columns.++        Parameters+        ----------+        other : DataFrame or Series/dict-like object, or list of these+            The data to append.+        ignore_index : boolean, default False+            If True, do not use the index labels.+        verify_integrity : boolean, default False+            If True, raise ValueError on creating index with duplicates.+        sort : boolean, default None+            Sort columns if the columns of `self` and `other` are not aligned.+            The default sorting is deprecated and will change to not-sorting+            in a future version of pandas. Explicitly pass ``sort=True`` to+            silence the warning and sort. Explicitly pass ``sort=False`` to+            silence the warning and not sort.++            .. versionadded:: 0.23.0++        Returns+        -------+        appended : DataFrame++        Notes+        -----+        If a list of dict/series is passed and the keys are all contained in+        the DataFrame's index, the order of the columns in the resulting+        DataFrame will be unchanged.++        Iteratively appending rows to a DataFrame can be more computationally+        intensive than a single concatenate. A better solution is to append+        those rows to a list and then concatenate the list with the original+        DataFrame all at once.++        See also+        --------+        pandas.concat : General function to concatenate DataFrame, Series+            or Panel objects++        Examples+        --------++        >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=list('AB'))+        >>> df+           A  B+        0  1  2+        1  3  4+        >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=list('AB'))+        >>> df.append(df2)+           A  B+        0  1  2+        1  3  4+        0  5  6+        1  7  8++        With `ignore_index` set to True:++        >>> df.append(df2, ignore_index=True)+           A  B+        0  1  2+        1  3  4+        2  5  6+        3  7  8++        The following, while not recommended methods for generating DataFrames,+        show two ways to generate a DataFrame from multiple data sources.++        Less efficient:++        >>> df = pd.DataFrame(columns=['A'])+        >>> for i in range(5):+        ...     df = df.append({'A': i}, ignore_index=True)+        >>> df+           A+        0  0+        1  1+        2  2+        3  3+        4  4++        More efficient:++        >>> pd.concat([pd.DataFrame([i], columns=['A']) for i in range(5)],+        ...           ignore_index=True)+           A+        0  0+        1  1+        2  2+        3  3+        4  4+        """+        if isinstance(other, (Series, dict)):+            if isinstance(other, dict):+                other = Series(other)+            if other.name is None and not ignore_index:+                raise TypeError('Can only append a Series if ignore_index=True'+                                ' or if the Series has a name')++            if other.name is None:+                index = None+            else:+                # other must have the same index name as self, otherwise+                # index name will be reset+                index = Index([other.name], name=self.index.name)++            idx_diff = other.index.difference(self.columns)+            try:+                combined_columns = self.columns.append(idx_diff)+            except TypeError:+                combined_columns = self.columns.astype(object).append(idx_diff)+            other = other.reindex(combined_columns, copy=False)+            other = DataFrame(other.values.reshape((1, len(other))),+                              index=index,+                              columns=combined_columns)+            other = other._convert(datetime=True, timedelta=True)+            if not self.columns.equals(combined_columns):+                self = self.reindex(columns=combined_columns)+        elif isinstance(other, list) and not isinstance(other[0], DataFrame):+            other = DataFrame(other)+            if (self.columns.get_indexer(other.columns) >= 0).all():+                other = other.loc[:, self.columns]++        from pandas.core.reshape.concat import concat+        if isinstance(other, (list, tuple)):+            to_concat = [self] + other+        else:+            to_concat = [self, other]+        return concat(to_concat, ignore_index=ignore_index,+                      verify_integrity=verify_integrity,+                      sort=sort)++    def join(self, other, on=None, how='left', lsuffix='', rsuffix='',+             sort=False):+        """+        Join columns with other DataFrame either on index or on a key+        column. Efficiently Join multiple DataFrame objects by index at once by+        passing a list.++        Parameters+        ----------+        other : DataFrame, Series with name field set, or list of DataFrame+            Index should be similar to one of the columns in this one. If a+            Series is passed, its name attribute must be set, and that will be+            used as the column name in the resulting joined DataFrame+        on : name, tuple/list of names, or array-like+            Column or index level name(s) in the caller to join on the index+            in `other`, otherwise joins index-on-index. If multiple+            values given, the `other` DataFrame must have a MultiIndex. Can+            pass an array as the join key if it is not already contained in+            the calling DataFrame. Like an Excel VLOOKUP operation+        how : {'left', 'right', 'outer', 'inner'}, default: 'left'+            How to handle the operation of the two objects.++            * left: use calling frame's index (or column if on is specified)+            * right: use other frame's index+            * outer: form union of calling frame's index (or column if on is+              specified) with other frame's index, and sort it+              lexicographically+            * inner: form intersection of calling frame's index (or column if+              on is specified) with other frame's index, preserving the order+              of the calling's one+        lsuffix : string+            Suffix to use from left frame's overlapping columns+        rsuffix : string+            Suffix to use from right frame's overlapping columns+        sort : boolean, default False+            Order result DataFrame lexicographically by the join key. If False,+            the order of the join key depends on the join type (how keyword)++        Notes+        -----+        on, lsuffix, and rsuffix options are not supported when passing a list+        of DataFrame objects++        Support for specifying index levels as the `on` parameter was added+        in version 0.23.0++        Examples+        --------+        >>> caller = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3', 'K4', 'K5'],+        ...                        'A': ['A0', 'A1', 'A2', 'A3', 'A4', 'A5']})++        >>> caller+            A key+        0  A0  K0+        1  A1  K1+        2  A2  K2+        3  A3  K3+        4  A4  K4+        5  A5  K5++        >>> other = pd.DataFrame({'key': ['K0', 'K1', 'K2'],+        ...                       'B': ['B0', 'B1', 'B2']})++        >>> other+            B key+        0  B0  K0+        1  B1  K1+        2  B2  K2++        Join DataFrames using their indexes.++        >>> caller.join(other, lsuffix='_caller', rsuffix='_other')++        >>>     A key_caller    B key_other+            0  A0         K0   B0        K0+            1  A1         K1   B1        K1+            2  A2         K2   B2        K2+            3  A3         K3  NaN       NaN+            4  A4         K4  NaN       NaN+            5  A5         K5  NaN       NaN+++        If we want to join using the key columns, we need to set key to be+        the index in both caller and other. The joined DataFrame will have+        key as its index.++        >>> caller.set_index('key').join(other.set_index('key'))++        >>>      A    B+            key+            K0   A0   B0+            K1   A1   B1+            K2   A2   B2+            K3   A3  NaN+            K4   A4  NaN+            K5   A5  NaN++        Another option to join using the key columns is to use the on+        parameter. DataFrame.join always uses other's index but we can use any+        column in the caller. This method preserves the original caller's+        index in the result.++        >>> caller.join(other.set_index('key'), on='key')++        >>>     A key    B+            0  A0  K0   B0+            1  A1  K1   B1+            2  A2  K2   B2+            3  A3  K3  NaN+            4  A4  K4  NaN+            5  A5  K5  NaN+++        See also+        --------+        DataFrame.merge : For column(s)-on-columns(s) operations++        Returns+        -------+        joined : DataFrame+        """+        # For SparseDataFrame's benefit+        return self._join_compat(other, on=on, how=how, lsuffix=lsuffix,+                                 rsuffix=rsuffix, sort=sort)++    def _join_compat(self, other, on=None, how='left', lsuffix='', rsuffix='',+                     sort=False):+        from pandas.core.reshape.merge import merge+        from pandas.core.reshape.concat import concat++        if isinstance(other, Series):+            if other.name is None:+                raise ValueError('Other Series must have a name')+            other = DataFrame({other.name: other})++        if isinstance(other, DataFrame):+            return merge(self, other, left_on=on, how=how,+                         left_index=on is None, right_index=True,+                         suffixes=(lsuffix, rsuffix), sort=sort)+        else:+            if on is not None:+                raise ValueError('Joining multiple DataFrames only supported'+                                 ' for joining on index')++            frames = [self] + list(other)++            can_concat = all(df.index.is_unique for df in frames)++            # join indexes only using concat+            if can_concat:+                if how == 'left':+                    how = 'outer'+                    join_axes = [self.index]+                else:+                    join_axes = None+                return concat(frames, axis=1, join=how, join_axes=join_axes,+                              verify_integrity=True)++            joined = frames[0]++            for frame in frames[1:]:+                joined = merge(joined, frame, how=how, left_index=True,+                               right_index=True)++            return joined++    @Substitution('')+    @Appender(_merge_doc, indents=2)+    def merge(self, right, how='inner', on=None, left_on=None, right_on=None,+              left_index=False, right_index=False, sort=False,+              suffixes=('_x', '_y'), copy=True, indicator=False,+              validate=None):+        from pandas.core.reshape.merge import merge+        return merge(self, right, how=how, on=on, left_on=left_on,+                     right_on=right_on, left_index=left_index,+                     right_index=right_index, sort=sort, suffixes=suffixes,+                     copy=copy, indicator=indicator, validate=validate)++    def round(self, decimals=0, *args, **kwargs):+        """+        Round a DataFrame to a variable number of decimal places.++        Parameters+        ----------+        decimals : int, dict, Series+            Number of decimal places to round each column to. If an int is+            given, round each column to the same number of places.+            Otherwise dict and Series round to variable numbers of places.+            Column names should be in the keys if `decimals` is a+            dict-like, or in the index if `decimals` is a Series. Any+            columns not included in `decimals` will be left as is. Elements+            of `decimals` which are not columns of the input will be+            ignored.++        Examples+        --------+        >>> df = pd.DataFrame(np.random.random([3, 3]),+        ...     columns=['A', 'B', 'C'], index=['first', 'second', 'third'])+        >>> df+                       A         B         C+        first   0.028208  0.992815  0.173891+        second  0.038683  0.645646  0.577595+        third   0.877076  0.149370  0.491027+        >>> df.round(2)+                   A     B     C+        first   0.03  0.99  0.17+        second  0.04  0.65  0.58+        third   0.88  0.15  0.49+        >>> df.round({'A': 1, 'C': 2})+                  A         B     C+        first   0.0  0.992815  0.17+        second  0.0  0.645646  0.58+        third   0.9  0.149370  0.49+        >>> decimals = pd.Series([1, 0, 2], index=['A', 'B', 'C'])+        >>> df.round(decimals)+                  A  B     C+        first   0.0  1  0.17+        second  0.0  1  0.58+        third   0.9  0  0.49++        Returns+        -------+        DataFrame object++        See Also+        --------+        numpy.around+        Series.round+        """+        from pandas.core.reshape.concat import concat++        def _dict_round(df, decimals):+            for col, vals in df.iteritems():+                try:+                    yield _series_round(vals, decimals[col])+                except KeyError:+                    yield vals++        def _series_round(s, decimals):+            if is_integer_dtype(s) or is_float_dtype(s):+                return s.round(decimals)+            return s++        nv.validate_round(args, kwargs)++        if isinstance(decimals, (dict, Series)):+            if isinstance(decimals, Series):+                if not decimals.index.is_unique:+                    raise ValueError("Index of decimals must be unique")+            new_cols = [col for col in _dict_round(self, decimals)]+        elif is_integer(decimals):+            # Dispatch to Series.round+            new_cols = [_series_round(v, decimals)+                        for _, v in self.iteritems()]+        else:+            raise TypeError("decimals must be an integer, a dict-like or a "+                            "Series")++        if len(new_cols) > 0:+            return self._constructor(concat(new_cols, axis=1),+                                     index=self.index,+                                     columns=self.columns)+        else:+            return self++    # ----------------------------------------------------------------------+    # Statistical methods, etc.++    def corr(self, method='pearson', min_periods=1):+        """+        Compute pairwise correlation of columns, excluding NA/null values++        Parameters+        ----------+        method : {'pearson', 'kendall', 'spearman'}+            * pearson : standard correlation coefficient+            * kendall : Kendall Tau correlation coefficient+            * spearman : Spearman rank correlation+        min_periods : int, optional+            Minimum number of observations required per pair of columns+            to have a valid result. Currently only available for pearson+            and spearman correlation++        Returns+        -------+        y : DataFrame+        """+        numeric_df = self._get_numeric_data()+        cols = numeric_df.columns+        idx = cols.copy()+        mat = numeric_df.values++        if method == 'pearson':+            correl = libalgos.nancorr(ensure_float64(mat), minp=min_periods)+        elif method == 'spearman':+            correl = libalgos.nancorr_spearman(ensure_float64(mat),+                                               minp=min_periods)+        elif method == 'kendall':+            if min_periods is None:+                min_periods = 1+            mat = ensure_float64(mat).T+            corrf = nanops.get_corr_func(method)+            K = len(cols)+            correl = np.empty((K, K), dtype=float)+            mask = np.isfinite(mat)+            for i, ac in enumerate(mat):+                for j, bc in enumerate(mat):+                    if i > j:+                        continue++                    valid = mask[i] & mask[j]+                    if valid.sum() < min_periods:+                        c = np.nan+                    elif i == j:+                        c = 1.+                    elif not valid.all():+                        c = corrf(ac[valid], bc[valid])+                    else:+                        c = corrf(ac, bc)+                    correl[i, j] = c+                    correl[j, i] = c+        else:+            raise ValueError("method must be either 'pearson', "+                             "'spearman', or 'kendall', '{method}' "+                             "was supplied".format(method=method))++        return self._constructor(correl, index=idx, columns=cols)++    def cov(self, min_periods=None):+        """+        Compute pairwise covariance of columns, excluding NA/null values.++        Compute the pairwise covariance among the series of a DataFrame.+        The returned data frame is the `covariance matrix+        <https://en.wikipedia.org/wiki/Covariance_matrix>`__ of the columns+        of the DataFrame.++        Both NA and null values are automatically excluded from the+        calculation. (See the note below about bias from missing values.)+        A threshold can be set for the minimum number of+        observations for each value created. Comparisons with observations+        below this threshold will be returned as ``NaN``.++        This method is generally used for the analysis of time series data to+        understand the relationship between different measures+        across time.++        Parameters+        ----------+        min_periods : int, optional+            Minimum number of observations required per pair of columns+            to have a valid result.++        Returns+        -------+        DataFrame+            The covariance matrix of the series of the DataFrame.++        See Also+        --------+        pandas.Series.cov : compute covariance with another Series+        pandas.core.window.EWM.cov: exponential weighted sample covariance+        pandas.core.window.Expanding.cov : expanding sample covariance+        pandas.core.window.Rolling.cov : rolling sample covariance++        Notes+        -----+        Returns the covariance matrix of the DataFrame's time series.+        The covariance is normalized by N-1.++        For DataFrames that have Series that are missing data (assuming that+        data is `missing at random+        <https://en.wikipedia.org/wiki/Missing_data#Missing_at_random>`__)+        the returned covariance matrix will be an unbiased estimate+        of the variance and covariance between the member Series.++        However, for many applications this estimate may not be acceptable+        because the estimate covariance matrix is not guaranteed to be positive+        semi-definite. This could lead to estimate correlations having+        absolute values which are greater than one, and/or a non-invertible+        covariance matrix. See `Estimation of covariance matrices+        <http://en.wikipedia.org/w/index.php?title=Estimation_of_covariance_+        matrices>`__ for more details.++        Examples+        --------+        >>> df = pd.DataFrame([(1, 2), (0, 3), (2, 0), (1, 1)],+        ...                   columns=['dogs', 'cats'])+        >>> df.cov()+                  dogs      cats+        dogs  0.666667 -1.000000+        cats -1.000000  1.666667++        >>> np.random.seed(42)+        >>> df = pd.DataFrame(np.random.randn(1000, 5),+        ...                   columns=['a', 'b', 'c', 'd', 'e'])+        >>> df.cov()+                  a         b         c         d         e+        a  0.998438 -0.020161  0.059277 -0.008943  0.014144+        b -0.020161  1.059352 -0.008543 -0.024738  0.009826+        c  0.059277 -0.008543  1.010670 -0.001486 -0.000271+        d -0.008943 -0.024738 -0.001486  0.921297 -0.013692+        e  0.014144  0.009826 -0.000271 -0.013692  0.977795++        **Minimum number of periods**++        This method also supports an optional ``min_periods`` keyword+        that specifies the required minimum number of non-NA observations for+        each column pair in order to have a valid result:++        >>> np.random.seed(42)+        >>> df = pd.DataFrame(np.random.randn(20, 3),+        ...                   columns=['a', 'b', 'c'])+        >>> df.loc[df.index[:5], 'a'] = np.nan+        >>> df.loc[df.index[5:10], 'b'] = np.nan+        >>> df.cov(min_periods=12)+                  a         b         c+        a  0.316741       NaN -0.150812+        b       NaN  1.248003  0.191417+        c -0.150812  0.191417  0.895202+        """+        numeric_df = self._get_numeric_data()+        cols = numeric_df.columns+        idx = cols.copy()+        mat = numeric_df.values++        if notna(mat).all():+            if min_periods is not None and min_periods > len(mat):+                baseCov = np.empty((mat.shape[1], mat.shape[1]))+                baseCov.fill(np.nan)+            else:+                baseCov = np.cov(mat.T)+            baseCov = baseCov.reshape((len(cols), len(cols)))+        else:+            baseCov = libalgos.nancorr(ensure_float64(mat), cov=True,+                                       minp=min_periods)++        return self._constructor(baseCov, index=idx, columns=cols)++    def corrwith(self, other, axis=0, drop=False):+        """+        Compute pairwise correlation between rows or columns of two DataFrame+        objects.++        Parameters+        ----------+        other : DataFrame, Series+        axis : {0 or 'index', 1 or 'columns'}, default 0+            0 or 'index' to compute column-wise, 1 or 'columns' for row-wise+        drop : boolean, default False+            Drop missing indices from result, default returns union of all++        Returns+        -------+        correls : Series+        """+        axis = self._get_axis_number(axis)+        this = self._get_numeric_data()++        if isinstance(other, Series):+            return this.apply(other.corr, axis=axis)++        other = other._get_numeric_data()++        left, right = this.align(other, join='inner', copy=False)++        # mask missing values+        left = left + right * 0+        right = right + left * 0++        if axis == 1:+            left = left.T+            right = right.T++        # demeaned data+        ldem = left - left.mean()+        rdem = right - right.mean()++        num = (ldem * rdem).sum()+        dom = (left.count() - 1) * left.std() * right.std()++        correl = num / dom++        if not drop:+            raxis = 1 if axis == 0 else 0+            result_index = this._get_axis(raxis).union(other._get_axis(raxis))+            correl = correl.reindex(result_index)++        return correl++    # ----------------------------------------------------------------------+    # ndarray-like stats methods++    def count(self, axis=0, level=None, numeric_only=False):+        """+        Count non-NA cells for each column or row.++        The values `None`, `NaN`, `NaT`, and optionally `numpy.inf` (depending+        on `pandas.options.mode.use_inf_as_na`) are considered NA.++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns'}, default 0+            If 0 or 'index' counts are generated for each column.+            If 1 or 'columns' counts are generated for each **row**.+        level : int or str, optional+            If the axis is a `MultiIndex` (hierarchical), count along a+            particular `level`, collapsing into a `DataFrame`.+            A `str` specifies the level name.+        numeric_only : boolean, default False+            Include only `float`, `int` or `boolean` data.++        Returns+        -------+        Series or DataFrame+            For each column/row the number of non-NA/null entries.+            If `level` is specified returns a `DataFrame`.++        See Also+        --------+        Series.count: number of non-NA elements in a Series+        DataFrame.shape: number of DataFrame rows and columns (including NA+            elements)+        DataFrame.isna: boolean same-sized DataFrame showing places of NA+            elements++        Examples+        --------+        Constructing DataFrame from a dictionary:++        >>> df = pd.DataFrame({"Person":+        ...                    ["John", "Myla", "Lewis", "John", "Myla"],+        ...                    "Age": [24., np.nan, 21., 33, 26],+        ...                    "Single": [False, True, True, True, False]})+        >>> df+           Person   Age  Single+        0    John  24.0   False+        1    Myla   NaN    True+        2   Lewis  21.0    True+        3    John  33.0    True+        4    Myla  26.0   False++        Notice the uncounted NA values:++        >>> df.count()+        Person    5+        Age       4+        Single    5+        dtype: int64++        Counts for each **row**:++        >>> df.count(axis='columns')+        0    3+        1    2+        2    3+        3    3+        4    3+        dtype: int64++        Counts for one level of a `MultiIndex`:++        >>> df.set_index(["Person", "Single"]).count(level="Person")+                Age+        Person+        John      2+        Lewis     1+        Myla      1++        """+        axis = self._get_axis_number(axis)+        if level is not None:+            return self._count_level(level, axis=axis,+                                     numeric_only=numeric_only)++        if numeric_only:+            frame = self._get_numeric_data()+        else:+            frame = self++        # GH #423+        if len(frame._get_axis(axis)) == 0:+            result = Series(0, index=frame._get_agg_axis(axis))+        else:+            if frame._is_mixed_type or frame._data.any_extension_types:+                # the or any_extension_types is really only hit for single-+                # column frames with an extension array+                result = notna(frame).sum(axis=axis)+            else:+                # GH13407+                series_counts = notna(frame).sum(axis=axis)+                counts = series_counts.values+                result = Series(counts, index=frame._get_agg_axis(axis))++        return result.astype('int64')++    def _count_level(self, level, axis=0, numeric_only=False):+        if numeric_only:+            frame = self._get_numeric_data()+        else:+            frame = self++        count_axis = frame._get_axis(axis)+        agg_axis = frame._get_agg_axis(axis)++        if not isinstance(count_axis, MultiIndex):+            raise TypeError("Can only count levels on hierarchical "+                            "{ax}.".format(ax=self._get_axis_name(axis)))++        if frame._is_mixed_type:+            # Since we have mixed types, calling notna(frame.values) might+            # upcast everything to object+            mask = notna(frame).values+        else:+            # But use the speedup when we have homogeneous dtypes+            mask = notna(frame.values)++        if axis == 1:+            # We're transposing the mask rather than frame to avoid potential+            # upcasts to object, which induces a ~20x slowdown+            mask = mask.T++        if isinstance(level, compat.string_types):+            level = count_axis._get_level_number(level)++        level_index = count_axis.levels[level]+        labels = ensure_int64(count_axis.labels[level])+        counts = lib.count_level_2d(mask, labels, len(level_index), axis=0)++        result = DataFrame(counts, index=level_index, columns=agg_axis)++        if axis == 1:+            # Undo our earlier transpose+            return result.T+        else:+            return result++    def _reduce(self, op, name, axis=0, skipna=True, numeric_only=None,+                filter_type=None, **kwds):+        if axis is None and filter_type == 'bool':+            labels = None+            constructor = None+        else:+            # TODO: Make other agg func handle axis=None properly+            axis = self._get_axis_number(axis)+            labels = self._get_agg_axis(axis)+            constructor = self._constructor++        def f(x):+            return op(x, axis=axis, skipna=skipna, **kwds)++        # exclude timedelta/datetime unless we are uniform types+        if axis == 1 and self._is_mixed_type and self._is_datelike_mixed_type:+            numeric_only = True++        if numeric_only is None:+            try:+                values = self.values+                result = f(values)++                if (filter_type == 'bool' and is_object_dtype(values) and+                        axis is None):+                    # work around https://github.com/numpy/numpy/issues/10489+                    # TODO: combine with hasattr(result, 'dtype') further down+                    # hard since we don't have `values` down there.+                    result = np.bool_(result)+            except Exception as e:++                # try by-column first+                if filter_type is None and axis == 0:+                    try:++                        # this can end up with a non-reduction+                        # but not always. if the types are mixed+                        # with datelike then need to make sure a series++                        # we only end up here if we have not specified+                        # numeric_only and yet we have tried a+                        # column-by-column reduction, where we have mixed type.+                        # So let's just do what we can+                        from pandas.core.apply import frame_apply+                        opa = frame_apply(self,+                                          func=f,+                                          result_type='expand',+                                          ignore_failures=True)+                        result = opa.get_result()+                        if result.ndim == self.ndim:+                            result = result.iloc[0]+                        return result+                    except Exception:+                        pass++                if filter_type is None or filter_type == 'numeric':+                    data = self._get_numeric_data()+                elif filter_type == 'bool':+                    data = self._get_bool_data()+                else:  # pragma: no cover+                    e = NotImplementedError(+                        "Handling exception with filter_type {f} not"+                        "implemented.".format(f=filter_type))+                    raise_with_traceback(e)+                with np.errstate(all='ignore'):+                    result = f(data.values)+                labels = data._get_agg_axis(axis)+        else:+            if numeric_only:+                if filter_type is None or filter_type == 'numeric':+                    data = self._get_numeric_data()+                elif filter_type == 'bool':+                    data = self._get_bool_data()+                else:  # pragma: no cover+                    msg = ("Generating numeric_only data with filter_type {f}"+                           "not supported.".format(f=filter_type))+                    raise NotImplementedError(msg)+                values = data.values+                labels = data._get_agg_axis(axis)+            else:+                values = self.values+            result = f(values)++        if hasattr(result, 'dtype') and is_object_dtype(result.dtype):+            try:+                if filter_type is None or filter_type == 'numeric':+                    result = result.astype(np.float64)+                elif filter_type == 'bool' and notna(result).all():+                    result = result.astype(np.bool_)+            except (ValueError, TypeError):++                # try to coerce to the original dtypes item by item if we can+                if axis == 0:+                    result = coerce_to_dtypes(result, self.dtypes)++        if constructor is not None:+            result = Series(result, index=labels)+        return result++    def nunique(self, axis=0, dropna=True):+        """+        Return Series with number of distinct observations over requested+        axis.++        .. versionadded:: 0.20.0++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns'}, default 0+        dropna : boolean, default True+            Don't include NaN in the counts.++        Returns+        -------+        nunique : Series++        Examples+        --------+        >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': [1, 1, 1]})+        >>> df.nunique()+        A    3+        B    1++        >>> df.nunique(axis=1)+        0    1+        1    2+        2    2+        """+        return self.apply(Series.nunique, axis=axis, dropna=dropna)++    def idxmin(self, axis=0, skipna=True):+        """+        Return index of first occurrence of minimum over requested axis.+        NA/null values are excluded.++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns'}, default 0+            0 or 'index' for row-wise, 1 or 'columns' for column-wise+        skipna : boolean, default True+            Exclude NA/null values. If an entire row/column is NA, the result+            will be NA.++        Raises+        ------+        ValueError+            * If the row/column is empty++        Returns+        -------+        idxmin : Series++        Notes+        -----+        This method is the DataFrame version of ``ndarray.argmin``.++        See Also+        --------+        Series.idxmin+        """+        axis = self._get_axis_number(axis)+        indices = nanops.nanargmin(self.values, axis=axis, skipna=skipna)+        index = self._get_axis(axis)+        result = [index[i] if i >= 0 else np.nan for i in indices]+        return Series(result, index=self._get_agg_axis(axis))++    def idxmax(self, axis=0, skipna=True):+        """+        Return index of first occurrence of maximum over requested axis.+        NA/null values are excluded.++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns'}, default 0+            0 or 'index' for row-wise, 1 or 'columns' for column-wise+        skipna : boolean, default True+            Exclude NA/null values. If an entire row/column is NA, the result+            will be NA.++        Raises+        ------+        ValueError+            * If the row/column is empty++        Returns+        -------+        idxmax : Series++        Notes+        -----+        This method is the DataFrame version of ``ndarray.argmax``.++        See Also+        --------+        Series.idxmax+        """+        axis = self._get_axis_number(axis)+        indices = nanops.nanargmax(self.values, axis=axis, skipna=skipna)+        index = self._get_axis(axis)+        result = [index[i] if i >= 0 else np.nan for i in indices]+        return Series(result, index=self._get_agg_axis(axis))++    def _get_agg_axis(self, axis_num):+        """ let's be explicit about this """+        if axis_num == 0:+            return self.columns+        elif axis_num == 1:+            return self.index+        else:+            raise ValueError('Axis must be 0 or 1 (got %r)' % axis_num)++    def mode(self, axis=0, numeric_only=False, dropna=True):+        """+        Gets the mode(s) of each element along the axis selected. Adds a row+        for each mode per label, fills in gaps with nan.++        Note that there could be multiple values returned for the selected+        axis (when more than one item share the maximum frequency), which is+        the reason why a dataframe is returned. If you want to impute missing+        values with the mode in a dataframe ``df``, you can just do this:+        ``df.fillna(df.mode().iloc[0])``++        Parameters+        ----------+        axis : {0 or 'index', 1 or 'columns'}, default 0+            * 0 or 'index' : get mode of each column+            * 1 or 'columns' : get mode of each row+        numeric_only : boolean, default False+            if True, only apply to numeric columns+        dropna : boolean, default True+            Don't consider counts of NaN/NaT.++            .. versionadded:: 0.24.0++        Returns+        -------+        modes : DataFrame (sorted)++        Examples+        --------+        >>> df = pd.DataFrame({'A': [1, 2, 1, 2, 1, 2, 3]})+        >>> df.mode()+           A+        0  1+        1  2+        """+        data = self if not numeric_only else self._get_numeric_data()++        def f(s):+            return s.mode(dropna=dropna)++        return data.apply(f, axis=axis)++    def quantile(self, q=0.5, axis=0, numeric_only=True,+                 interpolation='linear'):+        """+        Return values at the given quantile over requested axis, a la+        numpy.percentile.++        Parameters+        ----------+        q : float or array-like, default 0.5 (50% quantile)+            0 <= q <= 1, the quantile(s) to compute+        axis : {0, 1, 'index', 'columns'} (default 0)+            0 or 'index' for row-wise, 1 or 'columns' for column-wise+        numeric_only : boolean, default True+            If False, the quantile of datetime and timedelta data will be+            computed as well+        interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}+            .. versionadded:: 0.18.0++            This optional parameter specifies the interpolation method to use,+            when the desired quantile lies between two data points `i` and `j`:++            * linear: `i + (j - i) * fraction`, where `fraction` is the+              fractional part of the index surrounded by `i` and `j`.+            * lower: `i`.+            * higher: `j`.+            * nearest: `i` or `j` whichever is nearest.+            * midpoint: (`i` + `j`) / 2.++        Returns+        -------+        quantiles : Series or DataFrame++            - If ``q`` is an array, a DataFrame will be returned where the+              index is ``q``, the columns are the columns of self, and the+              values are the quantiles.+            - If ``q`` is a float, a Series will be returned where the+              index is the columns of self and the values are the quantiles.++        Examples+        --------++        >>> df = pd.DataFrame(np.array([[1, 1], [2, 10], [3, 100], [4, 100]]),+                              columns=['a', 'b'])+        >>> df.quantile(.1)+        a    1.3+        b    3.7+        dtype: float64+        >>> df.quantile([.1, .5])+               a     b+        0.1  1.3   3.7+        0.5  2.5  55.0++        Specifying `numeric_only=False` will also compute the quantile of+        datetime and timedelta data.++        >>> df = pd.DataFrame({'A': [1, 2],+                               'B': [pd.Timestamp('2010'),+                                     pd.Timestamp('2011')],+                               'C': [pd.Timedelta('1 days'),+                                     pd.Timedelta('2 days')]})+        >>> df.quantile(0.5, numeric_only=False)+        A                    1.5+        B    2010-07-02 12:00:00+        C        1 days 12:00:00+        Name: 0.5, dtype: object++        See Also+        --------+        pandas.core.window.Rolling.quantile+        """+        self._check_percentile(q)++        data = self._get_numeric_data() if numeric_only else self+        axis = self._get_axis_number(axis)+        is_transposed = axis == 1++        if is_transposed:+            data = data.T++        result = data._data.quantile(qs=q,+                                     axis=1,+                                     interpolation=interpolation,+                                     transposed=is_transposed)++        if result.ndim == 2:+            result = self._constructor(result)+        else:+            result = self._constructor_sliced(result, name=q)++        if is_transposed:+            result = result.T++        return result++    def to_timestamp(self, freq=None, how='start', axis=0, copy=True):+        """+        Cast to DatetimeIndex of timestamps, at *beginning* of period++        Parameters+        ----------+        freq : string, default frequency of PeriodIndex+            Desired frequency+        how : {'s', 'e', 'start', 'end'}+            Convention for converting period to timestamp; start of period+            vs. end+        axis : {0 or 'index', 1 or 'columns'}, default 0+            The axis to convert (the index by default)+        copy : boolean, default True+            If false then underlying input data is not copied++        Returns+        -------+        df : DataFrame with DatetimeIndex+        """+        new_data = self._data+        if copy:+            new_data = new_data.copy()++        axis = self._get_axis_number(axis)+        if axis == 0:+            new_data.set_axis(1, self.index.to_timestamp(freq=freq, how=how))+        elif axis == 1:+            new_data.set_axis(0, self.columns.to_timestamp(freq=freq, how=how))+        else:  # pragma: no cover+            raise AssertionError('Axis must be 0 or 1. Got {ax!s}'.format(+                ax=axis))++        return self._constructor(new_data)++    def to_period(self, freq=None, axis=0, copy=True):+        """+        Convert DataFrame from DatetimeIndex to PeriodIndex with desired+        frequency (inferred from index if not passed)++        Parameters+        ----------+        freq : string, default+        axis : {0 or 'index', 1 or 'columns'}, default 0+            The axis to convert (the index by default)+        copy : boolean, default True+            If False then underlying input data is not copied++        Returns+        -------+        ts : TimeSeries with PeriodIndex+        """+        new_data = self._data+        if copy:+            new_data = new_data.copy()++        axis = self._get_axis_number(axis)+        if axis == 0:+            new_data.set_axis(1, self.index.to_period(freq=freq))+        elif axis == 1:+            new_data.set_axis(0, self.columns.to_period(freq=freq))+        else:  # pragma: no cover+            raise AssertionError('Axis must be 0 or 1. Got {ax!s}'.format(+                ax=axis))++        return self._constructor(new_data)++    def isin(self, values):+        """+        Return boolean DataFrame showing whether each element in the+        DataFrame is contained in values.++        Parameters+        ----------+        values : iterable, Series, DataFrame or dictionary+            The result will only be true at a location if all the+            labels match. If `values` is a Series, that's the index. If+            `values` is a dictionary, the keys must be the column names,+            which must match. If `values` is a DataFrame,+            then both the index and column labels must match.++        Returns+        -------++        DataFrame of booleans++        Examples+        --------+        When ``values`` is a list:++        >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', 'f']})+        >>> df.isin([1, 3, 12, 'a'])+               A      B+        0   True   True+        1  False  False+        2   True  False++        When ``values`` is a dict:++        >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': [1, 4, 7]})+        >>> df.isin({'A': [1, 3], 'B': [4, 7, 12]})+               A      B+        0   True  False  # Note that B didn't match the 1 here.+        1  False   True+        2   True   True++        When ``values`` is a Series or DataFrame:++        >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': ['a', 'b', 'f']})+        >>> df2 = pd.DataFrame({'A': [1, 3, 3, 2], 'B': ['e', 'f', 'f', 'e']})+        >>> df.isin(df2)+               A      B+        0   True  False+        1  False  False  # Column A in `df2` has a 3, but not at index 1.+        2   True   True+        """+        if isinstance(values, dict):+            from pandas.core.reshape.concat import concat+            values = collections.defaultdict(list, values)+            return concat((self.iloc[:, [i]].isin(values[col])+                           for i, col in enumerate(self.columns)), axis=1)+        elif isinstance(values, Series):+            if not values.index.is_unique:+                raise ValueError("cannot compute isin with "+                                 "a duplicate axis.")+            return self.eq(values.reindex_like(self), axis='index')+        elif isinstance(values, DataFrame):+            if not (values.columns.is_unique and values.index.is_unique):+                raise ValueError("cannot compute isin with "+                                 "a duplicate axis.")+            return self.eq(values.reindex_like(self))+        else:+            if not is_list_like(values):+                raise TypeError("only list-like or dict-like objects are "+                                "allowed to be passed to DataFrame.isin(), "+                                "you passed a "+                                "{0!r}".format(type(values).__name__))+            return DataFrame(+                algorithms.isin(self.values.ravel(),+                                values).reshape(self.shape), self.index,+                self.columns)++    # ----------------------------------------------------------------------+    # Add plotting methods to DataFrame+    plot = CachedAccessor("plot", gfx.FramePlotMethods)+    hist = gfx.hist_frame+    boxplot = gfx.boxplot_frame+++DataFrame._setup_axes(['index', 'columns'], info_axis=1, stat_axis=0,+                      axes_are_reversed=True, aliases={'rows': 0},+                      docs={+                          'index': 'The index (row labels) of the DataFrame.',+                          'columns': 'The column labels of the DataFrame.'})+DataFrame._add_numeric_operations()+DataFrame._add_series_or_dataframe_operations()++ops.add_flex_arithmetic_methods(DataFrame)+ops.add_special_arithmetic_methods(DataFrame)+++def _arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):+    """+    Segregate Series based on type and coerce into matrices.+    Needs to handle a lot of exceptional cases.+    """+    # figure out the index, if necessary+    if index is None:+        index = extract_index(arrays)+    else:+        index = ensure_index(index)++    # don't force copy because getting jammed in an ndarray anyway+    arrays = _homogenize(arrays, index, dtype)++    # from BlockManager perspective+    axes = [ensure_index(columns), index]++    return create_block_manager_from_arrays(arrays, arr_names, axes)+++def extract_index(data):+    from pandas.core.index import _union_indexes++    index = None+    if len(data) == 0:+        index = Index([])+    elif len(data) > 0:+        raw_lengths = []+        indexes = []++        have_raw_arrays = False+        have_series = False+        have_dicts = False++        for v in data:+            if isinstance(v, Series):+                have_series = True+                indexes.append(v.index)+            elif isinstance(v, dict):+                have_dicts = True+                indexes.append(list(v.keys()))+            elif is_list_like(v) and getattr(v, 'ndim', 1) == 1:+                have_raw_arrays = True+                raw_lengths.append(len(v))++        if not indexes and not raw_lengths:+            raise ValueError('If using all scalar values, you must pass'+                             ' an index')++        if have_series or have_dicts:+            index = _union_indexes(indexes)++        if have_raw_arrays:+            lengths = list(set(raw_lengths))+            if len(lengths) > 1:+                raise ValueError('arrays must all be same length')++            if have_dicts:+                raise ValueError('Mixing dicts with non-Series may lead to '+                                 'ambiguous ordering.')++            if have_series:+                if lengths[0] != len(index):+                    msg = ('array length %d does not match index length %d' %+                           (lengths[0], len(index)))+                    raise ValueError(msg)+            else:+                index = ibase.default_index(lengths[0])++    return ensure_index(index)+++def _prep_ndarray(values, copy=True):+    if not isinstance(values, (np.ndarray, Series, Index)):+        if len(values) == 0:+            return np.empty((0, 0), dtype=object)++        def convert(v):+            return maybe_convert_platform(v)++        # we could have a 1-dim or 2-dim list here+        # this is equiv of np.asarray, but does object conversion+        # and platform dtype preservation+        try:+            if is_list_like(values[0]) or hasattr(values[0], 'len'):+                values = np.array([convert(v) for v in values])+            elif isinstance(values[0], np.ndarray) and values[0].ndim == 0:+                # GH#21861+                values = np.array([convert(v) for v in values])+            else:+                values = convert(values)+        except:+            values = convert(values)++    else:++        # drop subclass info, do not copy data+        values = np.asarray(values)+        if copy:+            values = values.copy()++    if values.ndim == 1:+        values = values.reshape((values.shape[0], 1))+    elif values.ndim != 2:+        raise ValueError('Must pass 2-d input')++    return values+++def _to_arrays(data, columns, coerce_float=False, dtype=None):+    """+    Return list of arrays, columns+    """+    if isinstance(data, DataFrame):+        if columns is not None:+            arrays = [data._ixs(i, axis=1).values+                      for i, col in enumerate(data.columns) if col in columns]+        else:+            columns = data.columns+            arrays = [data._ixs(i, axis=1).values for i in range(len(columns))]++        return arrays, columns++    if not len(data):+        if isinstance(data, np.ndarray):+            columns = data.dtype.names+            if columns is not None:+                return [[]] * len(columns), columns+        return [], []  # columns if columns is not None else []+    if isinstance(data[0], (list, tuple)):+        return _list_to_arrays(data, columns, coerce_float=coerce_float,+                               dtype=dtype)+    elif isinstance(data[0], collections.Mapping):+        return _list_of_dict_to_arrays(data, columns,+                                       coerce_float=coerce_float, dtype=dtype)+    elif isinstance(data[0], Series):+        return _list_of_series_to_arrays(data, columns,+                                         coerce_float=coerce_float,+                                         dtype=dtype)+    elif isinstance(data[0], Categorical):+        if columns is None:+            columns = ibase.default_index(len(data))+        return data, columns+    elif (isinstance(data, (np.ndarray, Series, Index)) and+          data.dtype.names is not None):++        columns = list(data.dtype.names)+        arrays = [data[k] for k in columns]+        return arrays, columns+    else:+        # last ditch effort+        data = lmap(tuple, data)+        return _list_to_arrays(data, columns, coerce_float=coerce_float,+                               dtype=dtype)+++def _masked_rec_array_to_mgr(data, index, columns, dtype, copy):+    """ extract from a masked rec array and create the manager """++    # essentially process a record array then fill it+    fill_value = data.fill_value+    fdata = ma.getdata(data)+    if index is None:+        index = _get_names_from_index(fdata)+        if index is None:+            index = ibase.default_index(len(data))+    index = ensure_index(index)++    if columns is not None:+        columns = ensure_index(columns)+    arrays, arr_columns = _to_arrays(fdata, columns)++    # fill if needed+    new_arrays = []+    for fv, arr, col in zip(fill_value, arrays, arr_columns):+        mask = ma.getmaskarray(data[col])+        if mask.any():+            arr, fv = maybe_upcast(arr, fill_value=fv, copy=True)+            arr[mask] = fv+        new_arrays.append(arr)++    # create the manager+    arrays, arr_columns = _reorder_arrays(new_arrays, arr_columns, columns)+    if columns is None:+        columns = arr_columns++    mgr = _arrays_to_mgr(arrays, arr_columns, index, columns)++    if copy:+        mgr = mgr.copy()+    return mgr+++def _reorder_arrays(arrays, arr_columns, columns):+    # reorder according to the columns+    if (columns is not None and len(columns) and arr_columns is not None and+            len(arr_columns)):+        indexer = ensure_index(arr_columns).get_indexer(columns)+        arr_columns = ensure_index([arr_columns[i] for i in indexer])+        arrays = [arrays[i] for i in indexer]+    return arrays, arr_columns+++def _list_to_arrays(data, columns, coerce_float=False, dtype=None):+    if len(data) > 0 and isinstance(data[0], tuple):+        content = list(lib.to_object_array_tuples(data).T)+    else:+        # list of lists+        content = list(lib.to_object_array(data).T)+    return _convert_object_array(content, columns, dtype=dtype,+                                 coerce_float=coerce_float)+++def _list_of_series_to_arrays(data, columns, coerce_float=False, dtype=None):+    from pandas.core.index import _get_objs_combined_axis++    if columns is None:+        columns = _get_objs_combined_axis(data, sort=False)++    indexer_cache = {}++    aligned_values = []+    for s in data:+        index = getattr(s, 'index', None)+        if index is None:+            index = ibase.default_index(len(s))++        if id(index) in indexer_cache:+            indexer = indexer_cache[id(index)]+        else:+            indexer = indexer_cache[id(index)] = index.get_indexer(columns)++        values = com.values_from_object(s)+        aligned_values.append(algorithms.take_1d(values, indexer))++    values = np.vstack(aligned_values)++    if values.dtype == np.object_:+        content = list(values.T)+        return _convert_object_array(content, columns, dtype=dtype,+                                     coerce_float=coerce_float)+    else:+        return values.T, columns+++def _list_of_dict_to_arrays(data, columns, coerce_float=False, dtype=None):+    if columns is None:+        gen = (list(x.keys()) for x in data)+        sort = not any(isinstance(d, OrderedDict) for d in data)+        columns = lib.fast_unique_multiple_list_gen(gen, sort=sort)++    # assure that they are of the base dict class and not of derived+    # classes+    data = [(type(d) is dict) and d or dict(d) for d in data]++    content = list(lib.dicts_to_array(data, list(columns)).T)+    return _convert_object_array(content, columns, dtype=dtype,+                                 coerce_float=coerce_float)+++def _convert_object_array(content, columns, coerce_float=False, dtype=None):+    if columns is None:+        columns = ibase.default_index(len(content))+    else:+        if len(columns) != len(content):  # pragma: no cover+            # caller's responsibility to check for this...+            raise AssertionError('{col:d} columns passed, passed data had '+                                 '{con} columns'.format(col=len(columns),+                                                        con=len(content)))++    # provide soft conversion of object dtypes+    def convert(arr):+        if dtype != object and dtype != np.object:+            arr = lib.maybe_convert_objects(arr, try_float=coerce_float)+            arr = maybe_cast_to_datetime(arr, dtype)+        return arr++    arrays = [convert(arr) for arr in content]++    return arrays, columns+++def _get_names_from_index(data):+    has_some_name = any(getattr(s, 'name', None) is not None for s in data)+    if not has_some_name:+        return ibase.default_index(len(data))++    index = lrange(len(data))+    count = 0+    for i, s in enumerate(data):+        n = getattr(s, 'name', None)+        if n is not None:+            index[i] = n+        else:+            index[i] = 'Unnamed %d' % count+            count += 1++    return index+++def _homogenize(data, index, dtype=None):+    from pandas.core.series import _sanitize_array++    oindex = None+    homogenized = []++    for v in data:+        if isinstance(v, Series):+            if dtype is not None:+                v = v.astype(dtype)+            if v.index is not index:+                # Forces alignment. No need to copy data since we+                # are putting it into an ndarray later+                v = v.reindex(index, copy=False)+        else:+            if isinstance(v, dict):+                if oindex is None:+                    oindex = index.astype('O')++                if isinstance(index, (DatetimeIndex, TimedeltaIndex)):+                    v = com.dict_compat(v)+                else:+                    v = dict(v)+                v = lib.fast_multiget(v, oindex.values, default=np.nan)+            v = _sanitize_array(v, index, dtype=dtype, copy=False,+                                raise_cast_failure=False)++        homogenized.append(v)++    return homogenized+++def _from_nested_dict(data):+    # TODO: this should be seriously cythonized+    new_data = OrderedDict()+    for index, s in compat.iteritems(data):+        for col, v in compat.iteritems(s):+            new_data[col] = new_data.get(col, OrderedDict())+            new_data[col][index] = v+    return new_data+++def _put_str(s, space):+    return u'{s}'.format(s=s)[:space].ljust(space)
+ test/files/pypy.py view
@@ -0,0 +1,6399 @@+# Copyright (c) 2004 Python Software Foundation.+# All rights reserved.++# Written by Eric Price <eprice at tjhsst.edu>+#    and Facundo Batista <facundo at taniquetil.com.ar>+#    and Raymond Hettinger <python at rcn.com>+#    and Aahz <aahz at pobox.com>+#    and Tim Peters++# This module should be kept in sync with the latest updates of the+# IBM specification as it evolves.  Those updates will be treated+# as bug fixes (deviation from the spec is a compatibility, usability+# bug) and will be backported.  At this point the spec is stabilizing+# and the updates are becoming fewer, smaller, and less significant.++"""+This is an implementation of decimal floating point arithmetic based on+the General Decimal Arithmetic Specification:++    http://speleotrove.com/decimal/decarith.html++and IEEE standard 854-1987:++    http://en.wikipedia.org/wiki/IEEE_854-1987++Decimal floating point has finite precision with arbitrarily large bounds.++The purpose of this module is to support arithmetic using familiar+"schoolhouse" rules and to avoid some of the tricky representation+issues associated with binary floating point.  The package is especially+useful for financial applications or for contexts where users have+expectations that are at odds with binary floating point (for instance,+in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead+of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected+Decimal('0.00')).++Here are some examples of using the decimal module:++>>> from decimal import *+>>> setcontext(ExtendedContext)+>>> Decimal(0)+Decimal('0')+>>> Decimal('1')+Decimal('1')+>>> Decimal('-.0123')+Decimal('-0.0123')+>>> Decimal(123456)+Decimal('123456')+>>> Decimal('123.45e12345678')+Decimal('1.2345E+12345680')+>>> Decimal('1.33') + Decimal('1.27')+Decimal('2.60')+>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')+Decimal('-2.20')+>>> dig = Decimal(1)+>>> print(dig / Decimal(3))+0.333333333+>>> getcontext().prec = 18+>>> print(dig / Decimal(3))+0.333333333333333333+>>> print(dig.sqrt())+1+>>> print(Decimal(3).sqrt())+1.73205080756887729+>>> print(Decimal(3) ** 123)+4.85192780976896427E+58+>>> inf = Decimal(1) / Decimal(0)+>>> print(inf)+Infinity+>>> neginf = Decimal(-1) / Decimal(0)+>>> print(neginf)+-Infinity+>>> print(neginf + inf)+NaN+>>> print(neginf * inf)+-Infinity+>>> print(dig / 0)+Infinity+>>> getcontext().traps[DivisionByZero] = 1+>>> print(dig / 0)+Traceback (most recent call last):+  ...+  ...+  ...+decimal.DivisionByZero: x / 0+>>> c = Context()+>>> c.traps[InvalidOperation] = 0+>>> print(c.flags[InvalidOperation])+0+>>> c.divide(Decimal(0), Decimal(0))+Decimal('NaN')+>>> c.traps[InvalidOperation] = 1+>>> print(c.flags[InvalidOperation])+1+>>> c.flags[InvalidOperation] = 0+>>> print(c.flags[InvalidOperation])+0+>>> print(c.divide(Decimal(0), Decimal(0)))+Traceback (most recent call last):+  ...+  ...+  ...+decimal.InvalidOperation: 0 / 0+>>> print(c.flags[InvalidOperation])+1+>>> c.flags[InvalidOperation] = 0+>>> c.traps[InvalidOperation] = 0+>>> print(c.divide(Decimal(0), Decimal(0)))+NaN+>>> print(c.flags[InvalidOperation])+1+>>>+"""++__all__ = [+    # Two major classes+    'Decimal', 'Context',++    # Named tuple representation+    'DecimalTuple',++    # Contexts+    'DefaultContext', 'BasicContext', 'ExtendedContext',++    # Exceptions+    'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero',+    'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow',+    'FloatOperation',++    # Exceptional conditions that trigger InvalidOperation+    'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined',++    # Constants for use in setting up contexts+    'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING',+    'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP',++    # Functions for manipulating contexts+    'setcontext', 'getcontext', 'localcontext',++    # Limits for the C version for compatibility+    'MAX_PREC',  'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',++    # C version: compile time choice that enables the thread local context+    'HAVE_THREADS'+]++__xname__ = __name__    # sys.modules lookup (--without-threads)+__name__ = 'decimal'    # For pickling+__version__ = '1.70'    # Highest version of the spec this complies with+                        # See http://speleotrove.com/decimal/+__libmpdec_version__ = "2.4.1" # compatible libmpdec version++import math as _math+import numbers as _numbers+import sys++try:+    from collections import namedtuple as _namedtuple+    DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')+except ImportError:+    DecimalTuple = lambda *args: args++# Rounding+ROUND_DOWN = 'ROUND_DOWN'+ROUND_HALF_UP = 'ROUND_HALF_UP'+ROUND_HALF_EVEN = 'ROUND_HALF_EVEN'+ROUND_CEILING = 'ROUND_CEILING'+ROUND_FLOOR = 'ROUND_FLOOR'+ROUND_UP = 'ROUND_UP'+ROUND_HALF_DOWN = 'ROUND_HALF_DOWN'+ROUND_05UP = 'ROUND_05UP'++# Compatibility with the C version+HAVE_THREADS = True+if sys.maxsize == 2**63-1:+    MAX_PREC = 999999999999999999+    MAX_EMAX = 999999999999999999+    MIN_EMIN = -999999999999999999+else:+    MAX_PREC = 425000000+    MAX_EMAX = 425000000+    MIN_EMIN = -425000000++MIN_ETINY = MIN_EMIN - (MAX_PREC-1)++# Errors++class DecimalException(ArithmeticError):+    """Base exception class.++    Used exceptions derive from this.+    If an exception derives from another exception besides this (such as+    Underflow (Inexact, Rounded, Subnormal) that indicates that it is only+    called if the others are present.  This isn't actually used for+    anything, though.++    handle  -- Called when context._raise_error is called and the+               trap_enabler is not set.  First argument is self, second is the+               context.  More arguments can be given, those being after+               the explanation in _raise_error (For example,+               context._raise_error(NewError, '(-x)!', self._sign) would+               call NewError().handle(context, self._sign).)++    To define a new exception, it should be sufficient to have it derive+    from DecimalException.+    """+    def handle(self, context, *args):+        pass+++class Clamped(DecimalException):+    """Exponent of a 0 changed to fit bounds.++    This occurs and signals clamped if the exponent of a result has been+    altered in order to fit the constraints of a specific concrete+    representation.  This may occur when the exponent of a zero result would+    be outside the bounds of a representation, or when a large normal+    number would have an encoded exponent that cannot be represented.  In+    this latter case, the exponent is reduced to fit and the corresponding+    number of zero digits are appended to the coefficient ("fold-down").+    """++class InvalidOperation(DecimalException):+    """An invalid operation was performed.++    Various bad things cause this:++    Something creates a signaling NaN+    -INF + INF+    0 * (+-)INF+    (+-)INF / (+-)INF+    x % 0+    (+-)INF % x+    x._rescale( non-integer )+    sqrt(-x) , x > 0+    0 ** 0+    x ** (non-integer)+    x ** (+-)INF+    An operand is invalid++    The result of the operation after these is a quiet positive NaN,+    except when the cause is a signaling NaN, in which case the result is+    also a quiet NaN, but with the original sign, and an optional+    diagnostic information.+    """+    def handle(self, context, *args):+        if args:+            ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True)+            return ans._fix_nan(context)+        return _NaN++class ConversionSyntax(InvalidOperation):+    """Trying to convert badly formed string.++    This occurs and signals invalid-operation if a string is being+    converted to a number and it does not conform to the numeric string+    syntax.  The result is [0,qNaN].+    """+    def handle(self, context, *args):+        return _NaN++class DivisionByZero(DecimalException, ZeroDivisionError):+    """Division by 0.++    This occurs and signals division-by-zero if division of a finite number+    by zero was attempted (during a divide-integer or divide operation, or a+    power operation with negative right-hand operand), and the dividend was+    not zero.++    The result of the operation is [sign,inf], where sign is the exclusive+    or of the signs of the operands for divide, or is 1 for an odd power of+    -0, for power.+    """++    def handle(self, context, sign, *args):+        return _SignedInfinity[sign]++class DivisionImpossible(InvalidOperation):+    """Cannot perform the division adequately.++    This occurs and signals invalid-operation if the integer result of a+    divide-integer or remainder operation had too many digits (would be+    longer than precision).  The result is [0,qNaN].+    """++    def handle(self, context, *args):+        return _NaN++class DivisionUndefined(InvalidOperation, ZeroDivisionError):+    """Undefined result of division.++    This occurs and signals invalid-operation if division by zero was+    attempted (during a divide-integer, divide, or remainder operation), and+    the dividend is also zero.  The result is [0,qNaN].+    """++    def handle(self, context, *args):+        return _NaN++class Inexact(DecimalException):+    """Had to round, losing information.++    This occurs and signals inexact whenever the result of an operation is+    not exact (that is, it needed to be rounded and any discarded digits+    were non-zero), or if an overflow or underflow condition occurs.  The+    result in all cases is unchanged.++    The inexact signal may be tested (or trapped) to determine if a given+    operation (or sequence of operations) was inexact.+    """++class InvalidContext(InvalidOperation):+    """Invalid context.  Unknown rounding, for example.++    This occurs and signals invalid-operation if an invalid context was+    detected during an operation.  This can occur if contexts are not checked+    on creation and either the precision exceeds the capability of the+    underlying concrete representation or an unknown or unsupported rounding+    was specified.  These aspects of the context need only be checked when+    the values are required to be used.  The result is [0,qNaN].+    """++    def handle(self, context, *args):+        return _NaN++class Rounded(DecimalException):+    """Number got rounded (not  necessarily changed during rounding).++    This occurs and signals rounded whenever the result of an operation is+    rounded (that is, some zero or non-zero digits were discarded from the+    coefficient), or if an overflow or underflow condition occurs.  The+    result in all cases is unchanged.++    The rounded signal may be tested (or trapped) to determine if a given+    operation (or sequence of operations) caused a loss of precision.+    """++class Subnormal(DecimalException):+    """Exponent < Emin before rounding.++    This occurs and signals subnormal whenever the result of a conversion or+    operation is subnormal (that is, its adjusted exponent is less than+    Emin, before any rounding).  The result in all cases is unchanged.++    The subnormal signal may be tested (or trapped) to determine if a given+    or operation (or sequence of operations) yielded a subnormal result.+    """++class Overflow(Inexact, Rounded):+    """Numerical overflow.++    This occurs and signals overflow if the adjusted exponent of a result+    (from a conversion or from an operation that is not an attempt to divide+    by zero), after rounding, would be greater than the largest value that+    can be handled by the implementation (the value Emax).++    The result depends on the rounding mode:++    For round-half-up and round-half-even (and for round-half-down and+    round-up, if implemented), the result of the operation is [sign,inf],+    where sign is the sign of the intermediate result.  For round-down, the+    result is the largest finite number that can be represented in the+    current precision, with the sign of the intermediate result.  For+    round-ceiling, the result is the same as for round-down if the sign of+    the intermediate result is 1, or is [0,inf] otherwise.  For round-floor,+    the result is the same as for round-down if the sign of the intermediate+    result is 0, or is [1,inf] otherwise.  In all cases, Inexact and Rounded+    will also be raised.+    """++    def handle(self, context, sign, *args):+        if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN,+                                ROUND_HALF_DOWN, ROUND_UP):+            return _SignedInfinity[sign]+        if sign == 0:+            if context.rounding == ROUND_CEILING:+                return _SignedInfinity[sign]+            return _dec_from_triple(sign, '9'*context.prec,+                            context.Emax-context.prec+1)+        if sign == 1:+            if context.rounding == ROUND_FLOOR:+                return _SignedInfinity[sign]+            return _dec_from_triple(sign, '9'*context.prec,+                             context.Emax-context.prec+1)+++class Underflow(Inexact, Rounded, Subnormal):+    """Numerical underflow with result rounded to 0.++    This occurs and signals underflow if a result is inexact and the+    adjusted exponent of the result would be smaller (more negative) than+    the smallest value that can be handled by the implementation (the value+    Emin).  That is, the result is both inexact and subnormal.++    The result after an underflow will be a subnormal number rounded, if+    necessary, so that its exponent is not less than Etiny.  This may result+    in 0 with the sign of the intermediate result and an exponent of Etiny.++    In all cases, Inexact, Rounded, and Subnormal will also be raised.+    """++class FloatOperation(DecimalException, TypeError):+    """Enable stricter semantics for mixing floats and Decimals.++    If the signal is not trapped (default), mixing floats and Decimals is+    permitted in the Decimal() constructor, context.create_decimal() and+    all comparison operators. Both conversion and comparisons are exact.+    Any occurrence of a mixed operation is silently recorded by setting+    FloatOperation in the context flags.  Explicit conversions with+    Decimal.from_float() or context.create_decimal_from_float() do not+    set the flag.++    Otherwise (the signal is trapped), only equality comparisons and explicit+    conversions are silent. All other mixed operations raise FloatOperation.+    """++# List of public traps and flags+_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded,+            Underflow, InvalidOperation, Subnormal, FloatOperation]++# Map conditions (per the spec) to signals+_condition_map = {ConversionSyntax:InvalidOperation,+                  DivisionImpossible:InvalidOperation,+                  DivisionUndefined:InvalidOperation,+                  InvalidContext:InvalidOperation}++# Valid rounding modes+_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,+                   ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP)++##### Context Functions ##################################################++# The getcontext() and setcontext() function manage access to a thread-local+# current context.  Py2.4 offers direct support for thread locals.  If that+# is not available, use threading.current_thread() which is slower but will+# work for older Pythons.  If threads are not part of the build, create a+# mock threading object with threading.local() returning the module namespace.++try:+    import threading+except ImportError:+    # Python was compiled without threads; create a mock object instead+    class MockThreading(object):+        def local(self, sys=sys):+            return sys.modules[__xname__]+    threading = MockThreading()+    del MockThreading++try:+    threading.local++except AttributeError:++    # To fix reloading, force it to create a new context+    # Old contexts have different exceptions in their dicts, making problems.+    if hasattr(threading.current_thread(), '__decimal_context__'):+        del threading.current_thread().__decimal_context__++    def setcontext(context):+        """Set this thread's context to context."""+        if context in (DefaultContext, BasicContext, ExtendedContext):+            context = context.copy()+            context.clear_flags()+        threading.current_thread().__decimal_context__ = context++    def getcontext():+        """Returns this thread's context.++        If this thread does not yet have a context, returns+        a new context and sets this thread's context.+        New contexts are copies of DefaultContext.+        """+        try:+            return threading.current_thread().__decimal_context__+        except AttributeError:+            context = Context()+            threading.current_thread().__decimal_context__ = context+            return context++else:++    local = threading.local()+    if hasattr(local, '__decimal_context__'):+        del local.__decimal_context__++    def getcontext(_local=local):+        """Returns this thread's context.++        If this thread does not yet have a context, returns+        a new context and sets this thread's context.+        New contexts are copies of DefaultContext.+        """+        try:+            return _local.__decimal_context__+        except AttributeError:+            context = Context()+            _local.__decimal_context__ = context+            return context++    def setcontext(context, _local=local):+        """Set this thread's context to context."""+        if context in (DefaultContext, BasicContext, ExtendedContext):+            context = context.copy()+            context.clear_flags()+        _local.__decimal_context__ = context++    del threading, local        # Don't contaminate the namespace++def localcontext(ctx=None):+    """Return a context manager for a copy of the supplied context++    Uses a copy of the current context if no context is specified+    The returned context manager creates a local decimal context+    in a with statement:+        def sin(x):+             with localcontext() as ctx:+                 ctx.prec += 2+                 # Rest of sin calculation algorithm+                 # uses a precision 2 greater than normal+             return +s  # Convert result to normal precision++         def sin(x):+             with localcontext(ExtendedContext):+                 # Rest of sin calculation algorithm+                 # uses the Extended Context from the+                 # General Decimal Arithmetic Specification+             return +s  # Convert result to normal context++    >>> setcontext(DefaultContext)+    >>> print(getcontext().prec)+    28+    >>> with localcontext():+    ...     ctx = getcontext()+    ...     ctx.prec += 2+    ...     print(ctx.prec)+    ...+    30+    >>> with localcontext(ExtendedContext):+    ...     print(getcontext().prec)+    ...+    9+    >>> print(getcontext().prec)+    28+    """+    if ctx is None: ctx = getcontext()+    return _ContextManager(ctx)+++##### Decimal class #######################################################++# Do not subclass Decimal from numbers.Real and do not register it as such+# (because Decimals are not interoperable with floats).  See the notes in+# numbers.py for more detail.++class Decimal(object):+    """Floating point class for decimal arithmetic."""++    __slots__ = ('_exp','_int','_sign', '_is_special')+    # Generally, the value of the Decimal instance is given by+    #  (-1)**_sign * _int * 10**_exp+    # Special values are signified by _is_special == True++    # We're immutable, so use __new__ not __init__+    def __new__(cls, value="0", context=None):+        """Create a decimal point instance.++        >>> Decimal('3.14')              # string input+        Decimal('3.14')+        >>> Decimal((0, (3, 1, 4), -2))  # tuple (sign, digit_tuple, exponent)+        Decimal('3.14')+        >>> Decimal(314)                 # int+        Decimal('314')+        >>> Decimal(Decimal(314))        # another decimal instance+        Decimal('314')+        >>> Decimal('  3.14  \\n')        # leading and trailing whitespace okay+        Decimal('3.14')+        """++        # Note that the coefficient, self._int, is actually stored as+        # a string rather than as a tuple of digits.  This speeds up+        # the "digits to integer" and "integer to digits" conversions+        # that are used in almost every arithmetic operation on+        # Decimals.  This is an internal detail: the as_tuple function+        # and the Decimal constructor still deal with tuples of+        # digits.++        self = object.__new__(cls)++        # From a string+        # REs insist on real strings, so we can too.+        if isinstance(value, str):+            m = _parser(value.strip())+            if m is None:+                if context is None:+                    context = getcontext()+                return context._raise_error(ConversionSyntax,+                                "Invalid literal for Decimal: %r" % value)++            if m.group('sign') == "-":+                self._sign = 1+            else:+                self._sign = 0+            intpart = m.group('int')+            if intpart is not None:+                # finite number+                fracpart = m.group('frac') or ''+                exp = int(m.group('exp') or '0')+                self._int = str(int(intpart+fracpart))+                self._exp = exp - len(fracpart)+                self._is_special = False+            else:+                diag = m.group('diag')+                if diag is not None:+                    # NaN+                    self._int = str(int(diag or '0')).lstrip('0')+                    if m.group('signal'):+                        self._exp = 'N'+                    else:+                        self._exp = 'n'+                else:+                    # infinity+                    self._int = '0'+                    self._exp = 'F'+                self._is_special = True+            return self++        # From an integer+        if isinstance(value, int):+            if value >= 0:+                self._sign = 0+            else:+                self._sign = 1+            self._exp = 0+            self._int = str(abs(value))+            self._is_special = False+            return self++        # From another decimal+        if isinstance(value, Decimal):+            self._exp  = value._exp+            self._sign = value._sign+            self._int  = value._int+            self._is_special  = value._is_special+            return self++        # From an internal working value+        if isinstance(value, _WorkRep):+            self._sign = value.sign+            self._int = str(value.int)+            self._exp = int(value.exp)+            self._is_special = False+            return self++        # tuple/list conversion (possibly from as_tuple())+        if isinstance(value, (list,tuple)):+            if len(value) != 3:+                raise ValueError('Invalid tuple size in creation of Decimal '+                                 'from list or tuple.  The list or tuple '+                                 'should have exactly three elements.')+            # process sign.  The isinstance test rejects floats+            if not (isinstance(value[0], int) and value[0] in (0,1)):+                raise ValueError("Invalid sign.  The first value in the tuple "+                                 "should be an integer; either 0 for a "+                                 "positive number or 1 for a negative number.")+            self._sign = value[0]+            if value[2] == 'F':+                # infinity: value[1] is ignored+                self._int = '0'+                self._exp = value[2]+                self._is_special = True+            else:+                # process and validate the digits in value[1]+                digits = []+                for digit in value[1]:+                    if isinstance(digit, int) and 0 <= digit <= 9:+                        # skip leading zeros+                        if digits or digit != 0:+                            digits.append(digit)+                    else:+                        raise ValueError("The second value in the tuple must "+                                         "be composed of integers in the range "+                                         "0 through 9.")+                if value[2] in ('n', 'N'):+                    # NaN: digits form the diagnostic+                    self._int = ''.join(map(str, digits))+                    self._exp = value[2]+                    self._is_special = True+                elif isinstance(value[2], int):+                    # finite number: digits give the coefficient+                    self._int = ''.join(map(str, digits or [0]))+                    self._exp = value[2]+                    self._is_special = False+                else:+                    raise ValueError("The third value in the tuple must "+                                     "be an integer, or one of the "+                                     "strings 'F', 'n', 'N'.")+            return self++        if isinstance(value, float):+            if context is None:+                context = getcontext()+            context._raise_error(FloatOperation,+                "strict semantics for mixing floats and Decimals are "+                "enabled")+            value = Decimal.from_float(value)+            self._exp  = value._exp+            self._sign = value._sign+            self._int  = value._int+            self._is_special  = value._is_special+            return self++        raise TypeError("Cannot convert %r to Decimal" % value)++    @classmethod+    def from_float(cls, f):+        """Converts a float to a decimal number, exactly.++        Note that Decimal.from_float(0.1) is not the same as Decimal('0.1').+        Since 0.1 is not exactly representable in binary floating point, the+        value is stored as the nearest representable value which is+        0x1.999999999999ap-4.  The exact equivalent of the value in decimal+        is 0.1000000000000000055511151231257827021181583404541015625.++        >>> Decimal.from_float(0.1)+        Decimal('0.1000000000000000055511151231257827021181583404541015625')+        >>> Decimal.from_float(float('nan'))+        Decimal('NaN')+        >>> Decimal.from_float(float('inf'))+        Decimal('Infinity')+        >>> Decimal.from_float(-float('inf'))+        Decimal('-Infinity')+        >>> Decimal.from_float(-0.0)+        Decimal('-0')++        """+        if isinstance(f, int):                # handle integer inputs+            return cls(f)+        if not isinstance(f, float):+            raise TypeError("argument must be int or float.")+        if _math.isinf(f) or _math.isnan(f):+            return cls(repr(f))+        if _math.copysign(1.0, f) == 1.0:+            sign = 0+        else:+            sign = 1+        n, d = abs(f).as_integer_ratio()+        k = d.bit_length() - 1+        result = _dec_from_triple(sign, str(n*5**k), -k)+        if cls is Decimal:+            return result+        else:+            return cls(result)++    def _isnan(self):+        """Returns whether the number is not actually one.++        0 if a number+        1 if NaN+        2 if sNaN+        """+        if self._is_special:+            exp = self._exp+            if exp == 'n':+                return 1+            elif exp == 'N':+                return 2+        return 0++    def _isinfinity(self):+        """Returns whether the number is infinite++        0 if finite or not a number+        1 if +INF+        -1 if -INF+        """+        if self._exp == 'F':+            if self._sign:+                return -1+            return 1+        return 0++    def _check_nans(self, other=None, context=None):+        """Returns whether the number is not actually one.++        if self, other are sNaN, signal+        if self, other are NaN return nan+        return 0++        Done before operations.+        """++        self_is_nan = self._isnan()+        if other is None:+            other_is_nan = False+        else:+            other_is_nan = other._isnan()++        if self_is_nan or other_is_nan:+            if context is None:+                context = getcontext()++            if self_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        self)+            if other_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        other)+            if self_is_nan:+                return self._fix_nan(context)++            return other._fix_nan(context)+        return 0++    def _compare_check_nans(self, other, context):+        """Version of _check_nans used for the signaling comparisons+        compare_signal, __le__, __lt__, __ge__, __gt__.++        Signal InvalidOperation if either self or other is a (quiet+        or signaling) NaN.  Signaling NaNs take precedence over quiet+        NaNs.++        Return 0 if neither operand is a NaN.++        """+        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            if self.is_snan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving sNaN',+                                            self)+            elif other.is_snan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving sNaN',+                                            other)+            elif self.is_qnan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving NaN',+                                            self)+            elif other.is_qnan():+                return context._raise_error(InvalidOperation,+                                            'comparison involving NaN',+                                            other)+        return 0++    def __bool__(self):+        """Return True if self is nonzero; otherwise return False.++        NaNs and infinities are considered nonzero.+        """+        return self._is_special or self._int != '0'++    def _cmp(self, other):+        """Compare the two non-NaN decimal instances self and other.++        Returns -1 if self < other, 0 if self == other and 1+        if self > other.  This routine is for internal use only."""++        if self._is_special or other._is_special:+            self_inf = self._isinfinity()+            other_inf = other._isinfinity()+            if self_inf == other_inf:+                return 0+            elif self_inf < other_inf:+                return -1+            else:+                return 1++        # check for zeros;  Decimal('0') == Decimal('-0')+        if not self:+            if not other:+                return 0+            else:+                return -((-1)**other._sign)+        if not other:+            return (-1)**self._sign++        # If different signs, neg one is less+        if other._sign < self._sign:+            return -1+        if self._sign < other._sign:+            return 1++        self_adjusted = self.adjusted()+        other_adjusted = other.adjusted()+        if self_adjusted == other_adjusted:+            self_padded = self._int + '0'*(self._exp - other._exp)+            other_padded = other._int + '0'*(other._exp - self._exp)+            if self_padded == other_padded:+                return 0+            elif self_padded < other_padded:+                return -(-1)**self._sign+            else:+                return (-1)**self._sign+        elif self_adjusted > other_adjusted:+            return (-1)**self._sign+        else: # self_adjusted < other_adjusted+            return -((-1)**self._sign)++    # Note: The Decimal standard doesn't cover rich comparisons for+    # Decimals.  In particular, the specification is silent on the+    # subject of what should happen for a comparison involving a NaN.+    # We take the following approach:+    #+    #   == comparisons involving a quiet NaN always return False+    #   != comparisons involving a quiet NaN always return True+    #   == or != comparisons involving a signaling NaN signal+    #      InvalidOperation, and return False or True as above if the+    #      InvalidOperation is not trapped.+    #   <, >, <= and >= comparisons involving a (quiet or signaling)+    #      NaN signal InvalidOperation, and return False if the+    #      InvalidOperation is not trapped.+    #+    # This behavior is designed to conform as closely as possible to+    # that specified by IEEE 754.++    def __eq__(self, other, context=None):+        self, other = _convert_for_comparison(self, other, equality_op=True)+        if other is NotImplemented:+            return other+        if self._check_nans(other, context):+            return False+        return self._cmp(other) == 0++    def __lt__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) < 0++    def __le__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) <= 0++    def __gt__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) > 0++    def __ge__(self, other, context=None):+        self, other = _convert_for_comparison(self, other)+        if other is NotImplemented:+            return other+        ans = self._compare_check_nans(other, context)+        if ans:+            return False+        return self._cmp(other) >= 0++    def compare(self, other, context=None):+        """Compare self to other.  Return a decimal value:++        a or b is a NaN ==> Decimal('NaN')+        a < b           ==> Decimal('-1')+        a == b          ==> Decimal('0')+        a > b           ==> Decimal('1')+        """+        other = _convert_other(other, raiseit=True)++        # Compare(NaN, NaN) = NaN+        if (self._is_special or other and other._is_special):+            ans = self._check_nans(other, context)+            if ans:+                return ans++        return Decimal(self._cmp(other))++    def __hash__(self):+        """x.__hash__() <==> hash(x)"""++        # In order to make sure that the hash of a Decimal instance+        # agrees with the hash of a numerically equal integer, float+        # or Fraction, we follow the rules for numeric hashes outlined+        # in the documentation.  (See library docs, 'Built-in Types').+        if self._is_special:+            if self.is_snan():+                raise TypeError('Cannot hash a signaling NaN value.')+            elif self.is_nan():+                return _PyHASH_NAN+            else:+                if self._sign:+                    return -_PyHASH_INF+                else:+                    return _PyHASH_INF++        if self._exp >= 0:+            exp_hash = pow(10, self._exp, _PyHASH_MODULUS)+        else:+            exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS)+        hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS+        ans = hash_ if self >= 0 else -hash_+        return -2 if ans == -1 else ans++    def as_tuple(self):+        """Represents the number as a triple tuple.++        To show the internals exactly as they are.+        """+        return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp)++    def __repr__(self):+        """Represents the number as an instance of Decimal."""+        # Invariant:  eval(repr(d)) == d+        return "Decimal('%s')" % str(self)++    def __str__(self, eng=False, context=None):+        """Return string representation of the number in scientific notation.++        Captures all of the information in the underlying representation.+        """++        sign = ['', '-'][self._sign]+        if self._is_special:+            if self._exp == 'F':+                return sign + 'Infinity'+            elif self._exp == 'n':+                return sign + 'NaN' + self._int+            else: # self._exp == 'N'+                return sign + 'sNaN' + self._int++        # number of digits of self._int to left of decimal point+        leftdigits = self._exp + len(self._int)++        # dotplace is number of digits of self._int to the left of the+        # decimal point in the mantissa of the output string (that is,+        # after adjusting the exponent)+        if self._exp <= 0 and leftdigits > -6:+            # no exponent required+            dotplace = leftdigits+        elif not eng:+            # usual scientific notation: 1 digit on left of the point+            dotplace = 1+        elif self._int == '0':+            # engineering notation, zero+            dotplace = (leftdigits + 1) % 3 - 1+        else:+            # engineering notation, nonzero+            dotplace = (leftdigits - 1) % 3 + 1++        if dotplace <= 0:+            intpart = '0'+            fracpart = '.' + '0'*(-dotplace) + self._int+        elif dotplace >= len(self._int):+            intpart = self._int+'0'*(dotplace-len(self._int))+            fracpart = ''+        else:+            intpart = self._int[:dotplace]+            fracpart = '.' + self._int[dotplace:]+        if leftdigits == dotplace:+            exp = ''+        else:+            if context is None:+                context = getcontext()+            exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace)++        return sign + intpart + fracpart + exp++    def to_eng_string(self, context=None):+        """Convert to a string, using engineering notation if an exponent is needed.++        Engineering notation has an exponent which is a multiple of 3.  This+        can leave up to 3 digits to the left of the decimal place and may+        require the addition of either one or two trailing zeros.+        """+        return self.__str__(eng=True, context=context)++    def __neg__(self, context=None):+        """Returns a copy with the sign switched.++        Rounds, if it has reason.+        """+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        if context is None:+            context = getcontext()++        if not self and context.rounding != ROUND_FLOOR:+            # -Decimal('0') is Decimal('0'), not Decimal('-0'), except+            # in ROUND_FLOOR rounding mode.+            ans = self.copy_abs()+        else:+            ans = self.copy_negate()++        return ans._fix(context)++    def __pos__(self, context=None):+        """Returns a copy, unless it is a sNaN.++        Rounds the number (if more than precision digits)+        """+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        if context is None:+            context = getcontext()++        if not self and context.rounding != ROUND_FLOOR:+            # + (-0) = 0, except in ROUND_FLOOR rounding mode.+            ans = self.copy_abs()+        else:+            ans = Decimal(self)++        return ans._fix(context)++    def __abs__(self, round=True, context=None):+        """Returns the absolute value of self.++        If the keyword argument 'round' is false, do not round.  The+        expression self.__abs__(round=False) is equivalent to+        self.copy_abs().+        """+        if not round:+            return self.copy_abs()++        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        if self._sign:+            ans = self.__neg__(context=context)+        else:+            ans = self.__pos__(context=context)++        return ans++    def __add__(self, other, context=None):+        """Returns self + other.++        -INF + INF (or the reverse) cause InvalidOperation errors.+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context)+            if ans:+                return ans++            if self._isinfinity():+                # If both INF, same sign => same as both, opposite => error.+                if self._sign != other._sign and other._isinfinity():+                    return context._raise_error(InvalidOperation, '-INF + INF')+                return Decimal(self)+            if other._isinfinity():+                return Decimal(other)  # Can't both be infinity here++        exp = min(self._exp, other._exp)+        negativezero = 0+        if context.rounding == ROUND_FLOOR and self._sign != other._sign:+            # If the answer is 0, the sign should be negative, in this case.+            negativezero = 1++        if not self and not other:+            sign = min(self._sign, other._sign)+            if negativezero:+                sign = 1+            ans = _dec_from_triple(sign, '0', exp)+            ans = ans._fix(context)+            return ans+        if not self:+            exp = max(exp, other._exp - context.prec-1)+            ans = other._rescale(exp, context.rounding)+            ans = ans._fix(context)+            return ans+        if not other:+            exp = max(exp, self._exp - context.prec-1)+            ans = self._rescale(exp, context.rounding)+            ans = ans._fix(context)+            return ans++        op1 = _WorkRep(self)+        op2 = _WorkRep(other)+        op1, op2 = _normalize(op1, op2, context.prec)++        result = _WorkRep()+        if op1.sign != op2.sign:+            # Equal and opposite+            if op1.int == op2.int:+                ans = _dec_from_triple(negativezero, '0', exp)+                ans = ans._fix(context)+                return ans+            if op1.int < op2.int:+                op1, op2 = op2, op1+                # OK, now abs(op1) > abs(op2)+            if op1.sign == 1:+                result.sign = 1+                op1.sign, op2.sign = op2.sign, op1.sign+            else:+                result.sign = 0+                # So we know the sign, and op1 > 0.+        elif op1.sign == 1:+            result.sign = 1+            op1.sign, op2.sign = (0, 0)+        else:+            result.sign = 0+        # Now, op1 > abs(op2) > 0++        if op2.sign == 0:+            result.int = op1.int + op2.int+        else:+            result.int = op1.int - op2.int++        result.exp = op1.exp+        ans = Decimal(result)+        ans = ans._fix(context)+        return ans++    __radd__ = __add__++    def __sub__(self, other, context=None):+        """Return self - other"""+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context=context)+            if ans:+                return ans++        # self - other is computed as self + other.copy_negate()+        return self.__add__(other.copy_negate(), context=context)++    def __rsub__(self, other, context=None):+        """Return other - self"""+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        return other.__sub__(self, context=context)++    def __mul__(self, other, context=None):+        """Return self * other.++        (+-) INF * 0 (or its reverse) raise InvalidOperation.+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        resultsign = self._sign ^ other._sign++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context)+            if ans:+                return ans++            if self._isinfinity():+                if not other:+                    return context._raise_error(InvalidOperation, '(+-)INF * 0')+                return _SignedInfinity[resultsign]++            if other._isinfinity():+                if not self:+                    return context._raise_error(InvalidOperation, '0 * (+-)INF')+                return _SignedInfinity[resultsign]++        resultexp = self._exp + other._exp++        # Special case for multiplying by zero+        if not self or not other:+            ans = _dec_from_triple(resultsign, '0', resultexp)+            # Fixing in case the exponent is out of bounds+            ans = ans._fix(context)+            return ans++        # Special case for multiplying by power of 10+        if self._int == '1':+            ans = _dec_from_triple(resultsign, other._int, resultexp)+            ans = ans._fix(context)+            return ans+        if other._int == '1':+            ans = _dec_from_triple(resultsign, self._int, resultexp)+            ans = ans._fix(context)+            return ans++        op1 = _WorkRep(self)+        op2 = _WorkRep(other)++        ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp)+        ans = ans._fix(context)++        return ans+    __rmul__ = __mul__++    def __truediv__(self, other, context=None):+        """Return self / other."""+        other = _convert_other(other)+        if other is NotImplemented:+            return NotImplemented++        if context is None:+            context = getcontext()++        sign = self._sign ^ other._sign++        if self._is_special or other._is_special:+            ans = self._check_nans(other, context)+            if ans:+                return ans++            if self._isinfinity() and other._isinfinity():+                return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF')++            if self._isinfinity():+                return _SignedInfinity[sign]++            if other._isinfinity():+                context._raise_error(Clamped, 'Division by infinity')+                return _dec_from_triple(sign, '0', context.Etiny())++        # Special cases for zeroes+        if not other:+            if not self:+                return context._raise_error(DivisionUndefined, '0 / 0')+            return context._raise_error(DivisionByZero, 'x / 0', sign)++        if not self:+            exp = self._exp - other._exp+            coeff = 0+        else:+            # OK, so neither = 0, INF or NaN+            shift = len(other._int) - len(self._int) + context.prec + 1+            exp = self._exp - other._exp - shift+            op1 = _WorkRep(self)+            op2 = _WorkRep(other)+            if shift >= 0:+                coeff, remainder = divmod(op1.int * 10**shift, op2.int)+            else:+                coeff, remainder = divmod(op1.int, op2.int * 10**-shift)+            if remainder:+                # result is not exact; adjust to ensure correct rounding+                if coeff % 5 == 0:+                    coeff += 1+            else:+                # result is exact; get as close to ideal exponent as possible+                ideal_exp = self._exp - other._exp+                while exp < ideal_exp and coeff % 10 == 0:+                    coeff //= 10+                    exp += 1++        ans = _dec_from_triple(sign, str(coeff), exp)+        return ans._fix(context)++    def _divide(self, other, context):+        """Return (self // other, self % other), to context.prec precision.++        Assumes that neither self nor other is a NaN, that self is not+        infinite and that other is nonzero.+        """+        sign = self._sign ^ other._sign+        if other._isinfinity():+            ideal_exp = self._exp+        else:+            ideal_exp = min(self._exp, other._exp)++        expdiff = self.adjusted() - other.adjusted()+        if not self or other._isinfinity() or expdiff <= -2:+            return (_dec_from_triple(sign, '0', 0),+                    self._rescale(ideal_exp, context.rounding))+        if expdiff <= context.prec:+            op1 = _WorkRep(self)+            op2 = _WorkRep(other)+            if op1.exp >= op2.exp:+                op1.int *= 10**(op1.exp - op2.exp)+            else:+                op2.int *= 10**(op2.exp - op1.exp)+            q, r = divmod(op1.int, op2.int)+            if q < 10**context.prec:+                return (_dec_from_triple(sign, str(q), 0),+                        _dec_from_triple(self._sign, str(r), ideal_exp))++        # Here the quotient is too large to be representable+        ans = context._raise_error(DivisionImpossible,+                                   'quotient too large in //, % or divmod')+        return ans, ans++    def __rtruediv__(self, other, context=None):+        """Swaps self/other and returns __truediv__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__truediv__(self, context=context)++    def __divmod__(self, other, context=None):+        """+        Return (self // other, self % other)+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return (ans, ans)++        sign = self._sign ^ other._sign+        if self._isinfinity():+            if other._isinfinity():+                ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)')+                return ans, ans+            else:+                return (_SignedInfinity[sign],+                        context._raise_error(InvalidOperation, 'INF % x'))++        if not other:+            if not self:+                ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)')+                return ans, ans+            else:+                return (context._raise_error(DivisionByZero, 'x // 0', sign),+                        context._raise_error(InvalidOperation, 'x % 0'))++        quotient, remainder = self._divide(other, context)+        remainder = remainder._fix(context)+        return quotient, remainder++    def __rdivmod__(self, other, context=None):+        """Swaps self/other and returns __divmod__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__divmod__(self, context=context)++    def __mod__(self, other, context=None):+        """+        self % other+        """+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if self._isinfinity():+            return context._raise_error(InvalidOperation, 'INF % x')+        elif not other:+            if self:+                return context._raise_error(InvalidOperation, 'x % 0')+            else:+                return context._raise_error(DivisionUndefined, '0 % 0')++        remainder = self._divide(other, context)[1]+        remainder = remainder._fix(context)+        return remainder++    def __rmod__(self, other, context=None):+        """Swaps self/other and returns __mod__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__mod__(self, context=context)++    def remainder_near(self, other, context=None):+        """+        Remainder nearest to 0-  abs(remainder-near) <= other/2+        """+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        # self == +/-infinity -> InvalidOperation+        if self._isinfinity():+            return context._raise_error(InvalidOperation,+                                        'remainder_near(infinity, x)')++        # other == 0 -> either InvalidOperation or DivisionUndefined+        if not other:+            if self:+                return context._raise_error(InvalidOperation,+                                            'remainder_near(x, 0)')+            else:+                return context._raise_error(DivisionUndefined,+                                            'remainder_near(0, 0)')++        # other = +/-infinity -> remainder = self+        if other._isinfinity():+            ans = Decimal(self)+            return ans._fix(context)++        # self = 0 -> remainder = self, with ideal exponent+        ideal_exponent = min(self._exp, other._exp)+        if not self:+            ans = _dec_from_triple(self._sign, '0', ideal_exponent)+            return ans._fix(context)++        # catch most cases of large or small quotient+        expdiff = self.adjusted() - other.adjusted()+        if expdiff >= context.prec + 1:+            # expdiff >= prec+1 => abs(self/other) > 10**prec+            return context._raise_error(DivisionImpossible)+        if expdiff <= -2:+            # expdiff <= -2 => abs(self/other) < 0.1+            ans = self._rescale(ideal_exponent, context.rounding)+            return ans._fix(context)++        # adjust both arguments to have the same exponent, then divide+        op1 = _WorkRep(self)+        op2 = _WorkRep(other)+        if op1.exp >= op2.exp:+            op1.int *= 10**(op1.exp - op2.exp)+        else:+            op2.int *= 10**(op2.exp - op1.exp)+        q, r = divmod(op1.int, op2.int)+        # remainder is r*10**ideal_exponent; other is +/-op2.int *+        # 10**ideal_exponent.   Apply correction to ensure that+        # abs(remainder) <= abs(other)/2+        if 2*r + (q&1) > op2.int:+            r -= op2.int+            q += 1++        if q >= 10**context.prec:+            return context._raise_error(DivisionImpossible)++        # result has same sign as self unless r is negative+        sign = self._sign+        if r < 0:+            sign = 1-sign+            r = -r++        ans = _dec_from_triple(sign, str(r), ideal_exponent)+        return ans._fix(context)++    def __floordiv__(self, other, context=None):+        """self // other"""+        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if self._isinfinity():+            if other._isinfinity():+                return context._raise_error(InvalidOperation, 'INF // INF')+            else:+                return _SignedInfinity[self._sign ^ other._sign]++        if not other:+            if self:+                return context._raise_error(DivisionByZero, 'x // 0',+                                            self._sign ^ other._sign)+            else:+                return context._raise_error(DivisionUndefined, '0 // 0')++        return self._divide(other, context)[0]++    def __rfloordiv__(self, other, context=None):+        """Swaps self/other and returns __floordiv__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__floordiv__(self, context=context)++    def __float__(self):+        """Float representation."""+        if self._isnan():+            if self.is_snan():+                raise ValueError("Cannot convert signaling NaN to float")+            s = "-nan" if self._sign else "nan"+        else:+            s = str(self)+        return float(s)++    def __int__(self):+        """Converts self to an int, truncating if necessary."""+        if self._is_special:+            if self._isnan():+                raise ValueError("Cannot convert NaN to integer")+            elif self._isinfinity():+                raise OverflowError("Cannot convert infinity to integer")+        s = (-1)**self._sign+        if self._exp >= 0:+            return s*int(self._int)*10**self._exp+        else:+            return s*int(self._int[:self._exp] or '0')++    __trunc__ = __int__++    def real(self):+        return self+    real = property(real)++    def imag(self):+        return Decimal(0)+    imag = property(imag)++    def conjugate(self):+        return self++    def __complex__(self):+        return complex(float(self))++    def _fix_nan(self, context):+        """Decapitate the payload of a NaN to fit the context"""+        payload = self._int++        # maximum length of payload is precision if clamp=0,+        # precision-1 if clamp=1.+        max_payload_len = context.prec - context.clamp+        if len(payload) > max_payload_len:+            payload = payload[len(payload)-max_payload_len:].lstrip('0')+            return _dec_from_triple(self._sign, payload, self._exp, True)+        return Decimal(self)++    def _fix(self, context):+        """Round if it is necessary to keep self within prec precision.++        Rounds and fixes the exponent.  Does not raise on a sNaN.++        Arguments:+        self - Decimal instance+        context - context used.+        """++        if self._is_special:+            if self._isnan():+                # decapitate payload if necessary+                return self._fix_nan(context)+            else:+                # self is +/-Infinity; return unaltered+                return Decimal(self)++        # if self is zero then exponent should be between Etiny and+        # Emax if clamp==0, and between Etiny and Etop if clamp==1.+        Etiny = context.Etiny()+        Etop = context.Etop()+        if not self:+            exp_max = [context.Emax, Etop][context.clamp]+            new_exp = min(max(self._exp, Etiny), exp_max)+            if new_exp != self._exp:+                context._raise_error(Clamped)+                return _dec_from_triple(self._sign, '0', new_exp)+            else:+                return Decimal(self)++        # exp_min is the smallest allowable exponent of the result,+        # equal to max(self.adjusted()-context.prec+1, Etiny)+        exp_min = len(self._int) + self._exp - context.prec+        if exp_min > Etop:+            # overflow: exp_min > Etop iff self.adjusted() > Emax+            ans = context._raise_error(Overflow, 'above Emax', self._sign)+            context._raise_error(Inexact)+            context._raise_error(Rounded)+            return ans++        self_is_subnormal = exp_min < Etiny+        if self_is_subnormal:+            exp_min = Etiny++        # round if self has too many digits+        if self._exp < exp_min:+            digits = len(self._int) + self._exp - exp_min+            if digits < 0:+                self = _dec_from_triple(self._sign, '1', exp_min-1)+                digits = 0+            rounding_method = self._pick_rounding_function[context.rounding]+            changed = rounding_method(self, digits)+            coeff = self._int[:digits] or '0'+            if changed > 0:+                coeff = str(int(coeff)+1)+                if len(coeff) > context.prec:+                    coeff = coeff[:-1]+                    exp_min += 1++            # check whether the rounding pushed the exponent out of range+            if exp_min > Etop:+                ans = context._raise_error(Overflow, 'above Emax', self._sign)+            else:+                ans = _dec_from_triple(self._sign, coeff, exp_min)++            # raise the appropriate signals, taking care to respect+            # the precedence described in the specification+            if changed and self_is_subnormal:+                context._raise_error(Underflow)+            if self_is_subnormal:+                context._raise_error(Subnormal)+            if changed:+                context._raise_error(Inexact)+            context._raise_error(Rounded)+            if not ans:+                # raise Clamped on underflow to 0+                context._raise_error(Clamped)+            return ans++        if self_is_subnormal:+            context._raise_error(Subnormal)++        # fold down if clamp == 1 and self has too few digits+        if context.clamp == 1 and self._exp > Etop:+            context._raise_error(Clamped)+            self_padded = self._int + '0'*(self._exp - Etop)+            return _dec_from_triple(self._sign, self_padded, Etop)++        # here self was representable to begin with; return unchanged+        return Decimal(self)++    # for each of the rounding functions below:+    #   self is a finite, nonzero Decimal+    #   prec is an integer satisfying 0 <= prec < len(self._int)+    #+    # each function returns either -1, 0, or 1, as follows:+    #   1 indicates that self should be rounded up (away from zero)+    #   0 indicates that self should be truncated, and that all the+    #     digits to be truncated are zeros (so the value is unchanged)+    #  -1 indicates that there are nonzero digits to be truncated++    def _round_down(self, prec):+        """Also known as round-towards-0, truncate."""+        if _all_zeros(self._int, prec):+            return 0+        else:+            return -1++    def _round_up(self, prec):+        """Rounds away from 0."""+        return -self._round_down(prec)++    def _round_half_up(self, prec):+        """Rounds 5 up (away from 0)"""+        if self._int[prec] in '56789':+            return 1+        elif _all_zeros(self._int, prec):+            return 0+        else:+            return -1++    def _round_half_down(self, prec):+        """Round 5 down"""+        if _exact_half(self._int, prec):+            return -1+        else:+            return self._round_half_up(prec)++    def _round_half_even(self, prec):+        """Round 5 to even, rest to nearest."""+        if _exact_half(self._int, prec) and \+                (prec == 0 or self._int[prec-1] in '02468'):+            return -1+        else:+            return self._round_half_up(prec)++    def _round_ceiling(self, prec):+        """Rounds up (not away from 0 if negative.)"""+        if self._sign:+            return self._round_down(prec)+        else:+            return -self._round_down(prec)++    def _round_floor(self, prec):+        """Rounds down (not towards 0 if negative)"""+        if not self._sign:+            return self._round_down(prec)+        else:+            return -self._round_down(prec)++    def _round_05up(self, prec):+        """Round down unless digit prec-1 is 0 or 5."""+        if prec and self._int[prec-1] not in '05':+            return self._round_down(prec)+        else:+            return -self._round_down(prec)++    _pick_rounding_function = dict(+        ROUND_DOWN = _round_down,+        ROUND_UP = _round_up,+        ROUND_HALF_UP = _round_half_up,+        ROUND_HALF_DOWN = _round_half_down,+        ROUND_HALF_EVEN = _round_half_even,+        ROUND_CEILING = _round_ceiling,+        ROUND_FLOOR = _round_floor,+        ROUND_05UP = _round_05up,+    )++    def __round__(self, n=None):+        """Round self to the nearest integer, or to a given precision.++        If only one argument is supplied, round a finite Decimal+        instance self to the nearest integer.  If self is infinite or+        a NaN then a Python exception is raised.  If self is finite+        and lies exactly halfway between two integers then it is+        rounded to the integer with even last digit.++        >>> round(Decimal('123.456'))+        123+        >>> round(Decimal('-456.789'))+        -457+        >>> round(Decimal('-3.0'))+        -3+        >>> round(Decimal('2.5'))+        2+        >>> round(Decimal('3.5'))+        4+        >>> round(Decimal('Inf'))+        Traceback (most recent call last):+          ...+        OverflowError: cannot round an infinity+        >>> round(Decimal('NaN'))+        Traceback (most recent call last):+          ...+        ValueError: cannot round a NaN++        If a second argument n is supplied, self is rounded to n+        decimal places using the rounding mode for the current+        context.++        For an integer n, round(self, -n) is exactly equivalent to+        self.quantize(Decimal('1En')).++        >>> round(Decimal('123.456'), 0)+        Decimal('123')+        >>> round(Decimal('123.456'), 2)+        Decimal('123.46')+        >>> round(Decimal('123.456'), -2)+        Decimal('1E+2')+        >>> round(Decimal('-Infinity'), 37)+        Decimal('NaN')+        >>> round(Decimal('sNaN123'), 0)+        Decimal('NaN123')++        """+        if n is not None:+            # two-argument form: use the equivalent quantize call+            if not isinstance(n, int):+                raise TypeError('Second argument to round should be integral')+            exp = _dec_from_triple(0, '1', -n)+            return self.quantize(exp)++        # one-argument form+        if self._is_special:+            if self.is_nan():+                raise ValueError("cannot round a NaN")+            else:+                raise OverflowError("cannot round an infinity")+        return int(self._rescale(0, ROUND_HALF_EVEN))++    def __floor__(self):+        """Return the floor of self, as an integer.++        For a finite Decimal instance self, return the greatest+        integer n such that n <= self.  If self is infinite or a NaN+        then a Python exception is raised.++        """+        if self._is_special:+            if self.is_nan():+                raise ValueError("cannot round a NaN")+            else:+                raise OverflowError("cannot round an infinity")+        return int(self._rescale(0, ROUND_FLOOR))++    def __ceil__(self):+        """Return the ceiling of self, as an integer.++        For a finite Decimal instance self, return the least integer n+        such that n >= self.  If self is infinite or a NaN then a+        Python exception is raised.++        """+        if self._is_special:+            if self.is_nan():+                raise ValueError("cannot round a NaN")+            else:+                raise OverflowError("cannot round an infinity")+        return int(self._rescale(0, ROUND_CEILING))++    def fma(self, other, third, context=None):+        """Fused multiply-add.++        Returns self*other+third with no rounding of the intermediate+        product self*other.++        self and other are multiplied together, with no rounding of+        the result.  The third operand is then added to the result,+        and a single final rounding is performed.+        """++        other = _convert_other(other, raiseit=True)+        third = _convert_other(third, raiseit=True)++        # compute product; raise InvalidOperation if either operand is+        # a signaling NaN or if the product is zero times infinity.+        if self._is_special or other._is_special:+            if context is None:+                context = getcontext()+            if self._exp == 'N':+                return context._raise_error(InvalidOperation, 'sNaN', self)+            if other._exp == 'N':+                return context._raise_error(InvalidOperation, 'sNaN', other)+            if self._exp == 'n':+                product = self+            elif other._exp == 'n':+                product = other+            elif self._exp == 'F':+                if not other:+                    return context._raise_error(InvalidOperation,+                                                'INF * 0 in fma')+                product = _SignedInfinity[self._sign ^ other._sign]+            elif other._exp == 'F':+                if not self:+                    return context._raise_error(InvalidOperation,+                                                '0 * INF in fma')+                product = _SignedInfinity[self._sign ^ other._sign]+        else:+            product = _dec_from_triple(self._sign ^ other._sign,+                                       str(int(self._int) * int(other._int)),+                                       self._exp + other._exp)++        return product.__add__(third, context)++    def _power_modulo(self, other, modulo, context=None):+        """Three argument version of __pow__"""++        other = _convert_other(other)+        if other is NotImplemented:+            return other+        modulo = _convert_other(modulo)+        if modulo is NotImplemented:+            return modulo++        if context is None:+            context = getcontext()++        # deal with NaNs: if there are any sNaNs then first one wins,+        # (i.e. behaviour for NaNs is identical to that of fma)+        self_is_nan = self._isnan()+        other_is_nan = other._isnan()+        modulo_is_nan = modulo._isnan()+        if self_is_nan or other_is_nan or modulo_is_nan:+            if self_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        self)+            if other_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        other)+            if modulo_is_nan == 2:+                return context._raise_error(InvalidOperation, 'sNaN',+                                        modulo)+            if self_is_nan:+                return self._fix_nan(context)+            if other_is_nan:+                return other._fix_nan(context)+            return modulo._fix_nan(context)++        # check inputs: we apply same restrictions as Python's pow()+        if not (self._isinteger() and+                other._isinteger() and+                modulo._isinteger()):+            return context._raise_error(InvalidOperation,+                                        'pow() 3rd argument not allowed '+                                        'unless all arguments are integers')+        if other < 0:+            return context._raise_error(InvalidOperation,+                                        'pow() 2nd argument cannot be '+                                        'negative when 3rd argument specified')+        if not modulo:+            return context._raise_error(InvalidOperation,+                                        'pow() 3rd argument cannot be 0')++        # additional restriction for decimal: the modulus must be less+        # than 10**prec in absolute value+        if modulo.adjusted() >= context.prec:+            return context._raise_error(InvalidOperation,+                                        'insufficient precision: pow() 3rd '+                                        'argument must not have more than '+                                        'precision digits')++        # define 0**0 == NaN, for consistency with two-argument pow+        # (even though it hurts!)+        if not other and not self:+            return context._raise_error(InvalidOperation,+                                        'at least one of pow() 1st argument '+                                        'and 2nd argument must be nonzero ;'+                                        '0**0 is not defined')++        # compute sign of result+        if other._iseven():+            sign = 0+        else:+            sign = self._sign++        # convert modulo to a Python integer, and self and other to+        # Decimal integers (i.e. force their exponents to be >= 0)+        modulo = abs(int(modulo))+        base = _WorkRep(self.to_integral_value())+        exponent = _WorkRep(other.to_integral_value())++        # compute result using integer pow()+        base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo+        for i in range(exponent.exp):+            base = pow(base, 10, modulo)+        base = pow(base, exponent.int, modulo)++        return _dec_from_triple(sign, str(base), 0)++    def _power_exact(self, other, p):+        """Attempt to compute self**other exactly.++        Given Decimals self and other and an integer p, attempt to+        compute an exact result for the power self**other, with p+        digits of precision.  Return None if self**other is not+        exactly representable in p digits.++        Assumes that elimination of special cases has already been+        performed: self and other must both be nonspecial; self must+        be positive and not numerically equal to 1; other must be+        nonzero.  For efficiency, other._exp should not be too large,+        so that 10**abs(other._exp) is a feasible calculation."""++        # In the comments below, we write x for the value of self and y for the+        # value of other.  Write x = xc*10**xe and abs(y) = yc*10**ye, with xc+        # and yc positive integers not divisible by 10.++        # The main purpose of this method is to identify the *failure*+        # of x**y to be exactly representable with as little effort as+        # possible.  So we look for cheap and easy tests that+        # eliminate the possibility of x**y being exact.  Only if all+        # these tests are passed do we go on to actually compute x**y.++        # Here's the main idea.  Express y as a rational number m/n, with m and+        # n relatively prime and n>0.  Then for x**y to be exactly+        # representable (at *any* precision), xc must be the nth power of a+        # positive integer and xe must be divisible by n.  If y is negative+        # then additionally xc must be a power of either 2 or 5, hence a power+        # of 2**n or 5**n.+        #+        # There's a limit to how small |y| can be: if y=m/n as above+        # then:+        #+        #  (1) if xc != 1 then for the result to be representable we+        #      need xc**(1/n) >= 2, and hence also xc**|y| >= 2.  So+        #      if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <=+        #      2**(1/|y|), hence xc**|y| < 2 and the result is not+        #      representable.+        #+        #  (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1.  Hence if+        #      |y| < 1/|xe| then the result is not representable.+        #+        # Note that since x is not equal to 1, at least one of (1) and+        # (2) must apply.  Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) <+        # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye.+        #+        # There's also a limit to how large y can be, at least if it's+        # positive: the normalized result will have coefficient xc**y,+        # so if it's representable then xc**y < 10**p, and y <+        # p/log10(xc).  Hence if y*log10(xc) >= p then the result is+        # not exactly representable.++        # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye,+        # so |y| < 1/xe and the result is not representable.+        # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y|+        # < 1/nbits(xc).++        x = _WorkRep(self)+        xc, xe = x.int, x.exp+        while xc % 10 == 0:+            xc //= 10+            xe += 1++        y = _WorkRep(other)+        yc, ye = y.int, y.exp+        while yc % 10 == 0:+            yc //= 10+            ye += 1++        # case where xc == 1: result is 10**(xe*y), with xe*y+        # required to be an integer+        if xc == 1:+            xe *= yc+            # result is now 10**(xe * 10**ye);  xe * 10**ye must be integral+            while xe % 10 == 0:+                xe //= 10+                ye += 1+            if ye < 0:+                return None+            exponent = xe * 10**ye+            if y.sign == 1:+                exponent = -exponent+            # if other is a nonnegative integer, use ideal exponent+            if other._isinteger() and other._sign == 0:+                ideal_exponent = self._exp*int(other)+                zeros = min(exponent-ideal_exponent, p-1)+            else:+                zeros = 0+            return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros)++        # case where y is negative: xc must be either a power+        # of 2 or a power of 5.+        if y.sign == 1:+            last_digit = xc % 10+            if last_digit in (2,4,6,8):+                # quick test for power of 2+                if xc & -xc != xc:+                    return None+                # now xc is a power of 2; e is its exponent+                e = _nbits(xc)-1++                # We now have:+                #+                #   x = 2**e * 10**xe, e > 0, and y < 0.+                #+                # The exact result is:+                #+                #   x**y = 5**(-e*y) * 10**(e*y + xe*y)+                #+                # provided that both e*y and xe*y are integers.  Note that if+                # 5**(-e*y) >= 10**p, then the result can't be expressed+                # exactly with p digits of precision.+                #+                # Using the above, we can guard against large values of ye.+                # 93/65 is an upper bound for log(10)/log(5), so if+                #+                #   ye >= len(str(93*p//65))+                #+                # then+                #+                #   -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5),+                #+                # so 5**(-e*y) >= 10**p, and the coefficient of the result+                # can't be expressed in p digits.++                # emax >= largest e such that 5**e < 10**p.+                emax = p*93//65+                if ye >= len(str(emax)):+                    return None++                # Find -e*y and -xe*y; both must be integers+                e = _decimal_lshift_exact(e * yc, ye)+                xe = _decimal_lshift_exact(xe * yc, ye)+                if e is None or xe is None:+                    return None++                if e > emax:+                    return None+                xc = 5**e++            elif last_digit == 5:+                # e >= log_5(xc) if xc is a power of 5; we have+                # equality all the way up to xc=5**2658+                e = _nbits(xc)*28//65+                xc, remainder = divmod(5**e, xc)+                if remainder:+                    return None+                while xc % 5 == 0:+                    xc //= 5+                    e -= 1++                # Guard against large values of ye, using the same logic as in+                # the 'xc is a power of 2' branch.  10/3 is an upper bound for+                # log(10)/log(2).+                emax = p*10//3+                if ye >= len(str(emax)):+                    return None++                e = _decimal_lshift_exact(e * yc, ye)+                xe = _decimal_lshift_exact(xe * yc, ye)+                if e is None or xe is None:+                    return None++                if e > emax:+                    return None+                xc = 2**e+            else:+                return None++            if xc >= 10**p:+                return None+            xe = -e-xe+            return _dec_from_triple(0, str(xc), xe)++        # now y is positive; find m and n such that y = m/n+        if ye >= 0:+            m, n = yc*10**ye, 1+        else:+            if xe != 0 and len(str(abs(yc*xe))) <= -ye:+                return None+            xc_bits = _nbits(xc)+            if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:+                return None+            m, n = yc, 10**(-ye)+            while m % 2 == n % 2 == 0:+                m //= 2+                n //= 2+            while m % 5 == n % 5 == 0:+                m //= 5+                n //= 5++        # compute nth root of xc*10**xe+        if n > 1:+            # if 1 < xc < 2**n then xc isn't an nth power+            if xc != 1 and xc_bits <= n:+                return None++            xe, rem = divmod(xe, n)+            if rem != 0:+                return None++            # compute nth root of xc using Newton's method+            a = 1 << -(-_nbits(xc)//n) # initial estimate+            while True:+                q, r = divmod(xc, a**(n-1))+                if a <= q:+                    break+                else:+                    a = (a*(n-1) + q)//n+            if not (a == q and r == 0):+                return None+            xc = a++        # now xc*10**xe is the nth root of the original xc*10**xe+        # compute mth power of xc*10**xe++        # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m >+        # 10**p and the result is not representable.+        if xc > 1 and m > p*100//_log10_lb(xc):+            return None+        xc = xc**m+        xe *= m+        if xc > 10**p:+            return None++        # by this point the result *is* exactly representable+        # adjust the exponent to get as close as possible to the ideal+        # exponent, if necessary+        str_xc = str(xc)+        if other._isinteger() and other._sign == 0:+            ideal_exponent = self._exp*int(other)+            zeros = min(xe-ideal_exponent, p-len(str_xc))+        else:+            zeros = 0+        return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros)++    def __pow__(self, other, modulo=None, context=None):+        """Return self ** other [ % modulo].++        With two arguments, compute self**other.++        With three arguments, compute (self**other) % modulo.  For the+        three argument form, the following restrictions on the+        arguments hold:++         - all three arguments must be integral+         - other must be nonnegative+         - either self or other (or both) must be nonzero+         - modulo must be nonzero and must have at most p digits,+           where p is the context precision.++        If any of these restrictions is violated the InvalidOperation+        flag is raised.++        The result of pow(self, other, modulo) is identical to the+        result that would be obtained by computing (self**other) %+        modulo with unbounded precision, but is computed more+        efficiently.  It is always exact.+        """++        if modulo is not None:+            return self._power_modulo(other, modulo, context)++        other = _convert_other(other)+        if other is NotImplemented:+            return other++        if context is None:+            context = getcontext()++        # either argument is a NaN => result is NaN+        ans = self._check_nans(other, context)+        if ans:+            return ans++        # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity)+        if not other:+            if not self:+                return context._raise_error(InvalidOperation, '0 ** 0')+            else:+                return _One++        # result has sign 1 iff self._sign is 1 and other is an odd integer+        result_sign = 0+        if self._sign == 1:+            if other._isinteger():+                if not other._iseven():+                    result_sign = 1+            else:+                # -ve**noninteger = NaN+                # (-0)**noninteger = 0**noninteger+                if self:+                    return context._raise_error(InvalidOperation,+                        'x ** y with x negative and y not an integer')+            # negate self, without doing any unwanted rounding+            self = self.copy_negate()++        # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity+        if not self:+            if other._sign == 0:+                return _dec_from_triple(result_sign, '0', 0)+            else:+                return _SignedInfinity[result_sign]++        # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0+        if self._isinfinity():+            if other._sign == 0:+                return _SignedInfinity[result_sign]+            else:+                return _dec_from_triple(result_sign, '0', 0)++        # 1**other = 1, but the choice of exponent and the flags+        # depend on the exponent of self, and on whether other is a+        # positive integer, a negative integer, or neither+        if self == _One:+            if other._isinteger():+                # exp = max(self._exp*max(int(other), 0),+                # 1-context.prec) but evaluating int(other) directly+                # is dangerous until we know other is small (other+                # could be 1e999999999)+                if other._sign == 1:+                    multiplier = 0+                elif other > context.prec:+                    multiplier = context.prec+                else:+                    multiplier = int(other)++                exp = self._exp * multiplier+                if exp < 1-context.prec:+                    exp = 1-context.prec+                    context._raise_error(Rounded)+            else:+                context._raise_error(Inexact)+                context._raise_error(Rounded)+                exp = 1-context.prec++            return _dec_from_triple(result_sign, '1'+'0'*-exp, exp)++        # compute adjusted exponent of self+        self_adj = self.adjusted()++        # self ** infinity is infinity if self > 1, 0 if self < 1+        # self ** -infinity is infinity if self < 1, 0 if self > 1+        if other._isinfinity():+            if (other._sign == 0) == (self_adj < 0):+                return _dec_from_triple(result_sign, '0', 0)+            else:+                return _SignedInfinity[result_sign]++        # from here on, the result always goes through the call+        # to _fix at the end of this function.+        ans = None+        exact = False++        # crude test to catch cases of extreme overflow/underflow.  If+        # log10(self)*other >= 10**bound and bound >= len(str(Emax))+        # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence+        # self**other >= 10**(Emax+1), so overflow occurs.  The test+        # for underflow is similar.+        bound = self._log10_exp_bound() + other.adjusted()+        if (self_adj >= 0) == (other._sign == 0):+            # self > 1 and other +ve, or self < 1 and other -ve+            # possibility of overflow+            if bound >= len(str(context.Emax)):+                ans = _dec_from_triple(result_sign, '1', context.Emax+1)+        else:+            # self > 1 and other -ve, or self < 1 and other +ve+            # possibility of underflow to 0+            Etiny = context.Etiny()+            if bound >= len(str(-Etiny)):+                ans = _dec_from_triple(result_sign, '1', Etiny-1)++        # try for an exact result with precision +1+        if ans is None:+            ans = self._power_exact(other, context.prec + 1)+            if ans is not None:+                if result_sign == 1:+                    ans = _dec_from_triple(1, ans._int, ans._exp)+                exact = True++        # usual case: inexact result, x**y computed directly as exp(y*log(x))+        if ans is None:+            p = context.prec+            x = _WorkRep(self)+            xc, xe = x.int, x.exp+            y = _WorkRep(other)+            yc, ye = y.int, y.exp+            if y.sign == 1:+                yc = -yc++            # compute correctly rounded result:  start with precision +3,+            # then increase precision until result is unambiguously roundable+            extra = 3+            while True:+                coeff, exp = _dpower(xc, xe, yc, ye, p+extra)+                if coeff % (5*10**(len(str(coeff))-p-1)):+                    break+                extra += 3++            ans = _dec_from_triple(result_sign, str(coeff), exp)++        # unlike exp, ln and log10, the power function respects the+        # rounding mode; no need to switch to ROUND_HALF_EVEN here++        # There's a difficulty here when 'other' is not an integer and+        # the result is exact.  In this case, the specification+        # requires that the Inexact flag be raised (in spite of+        # exactness), but since the result is exact _fix won't do this+        # for us.  (Correspondingly, the Underflow signal should also+        # be raised for subnormal results.)  We can't directly raise+        # these signals either before or after calling _fix, since+        # that would violate the precedence for signals.  So we wrap+        # the ._fix call in a temporary context, and reraise+        # afterwards.+        if exact and not other._isinteger():+            # pad with zeros up to length context.prec+1 if necessary; this+            # ensures that the Rounded signal will be raised.+            if len(ans._int) <= context.prec:+                expdiff = context.prec + 1 - len(ans._int)+                ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff,+                                       ans._exp-expdiff)++            # create a copy of the current context, with cleared flags/traps+            newcontext = context.copy()+            newcontext.clear_flags()+            for exception in _signals:+                newcontext.traps[exception] = 0++            # round in the new context+            ans = ans._fix(newcontext)++            # raise Inexact, and if necessary, Underflow+            newcontext._raise_error(Inexact)+            if newcontext.flags[Subnormal]:+                newcontext._raise_error(Underflow)++            # propagate signals to the original context; _fix could+            # have raised any of Overflow, Underflow, Subnormal,+            # Inexact, Rounded, Clamped.  Overflow needs the correct+            # arguments.  Note that the order of the exceptions is+            # important here.+            if newcontext.flags[Overflow]:+                context._raise_error(Overflow, 'above Emax', ans._sign)+            for exception in Underflow, Subnormal, Inexact, Rounded, Clamped:+                if newcontext.flags[exception]:+                    context._raise_error(exception)++        else:+            ans = ans._fix(context)++        return ans++    def __rpow__(self, other, context=None):+        """Swaps self/other and returns __pow__."""+        other = _convert_other(other)+        if other is NotImplemented:+            return other+        return other.__pow__(self, context=context)++    def normalize(self, context=None):+        """Normalize- strip trailing 0s, change anything equal to 0 to 0e0"""++        if context is None:+            context = getcontext()++        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++        dup = self._fix(context)+        if dup._isinfinity():+            return dup++        if not dup:+            return _dec_from_triple(dup._sign, '0', 0)+        exp_max = [context.Emax, context.Etop()][context.clamp]+        end = len(dup._int)+        exp = dup._exp+        while dup._int[end-1] == '0' and exp < exp_max:+            exp += 1+            end -= 1+        return _dec_from_triple(dup._sign, dup._int[:end], exp)++    def quantize(self, exp, rounding=None, context=None):+        """Quantize self so its exponent is the same as that of exp.++        Similar to self._rescale(exp._exp) but with error checking.+        """+        exp = _convert_other(exp, raiseit=True)++        if context is None:+            context = getcontext()+        if rounding is None:+            rounding = context.rounding++        if self._is_special or exp._is_special:+            ans = self._check_nans(exp, context)+            if ans:+                return ans++            if exp._isinfinity() or self._isinfinity():+                if exp._isinfinity() and self._isinfinity():+                    return Decimal(self)  # if both are inf, it is OK+                return context._raise_error(InvalidOperation,+                                        'quantize with one INF')++        # exp._exp should be between Etiny and Emax+        if not (context.Etiny() <= exp._exp <= context.Emax):+            return context._raise_error(InvalidOperation,+                   'target exponent out of bounds in quantize')++        if not self:+            ans = _dec_from_triple(self._sign, '0', exp._exp)+            return ans._fix(context)++        self_adjusted = self.adjusted()+        if self_adjusted > context.Emax:+            return context._raise_error(InvalidOperation,+                                        'exponent of quantize result too large for current context')+        if self_adjusted - exp._exp + 1 > context.prec:+            return context._raise_error(InvalidOperation,+                                        'quantize result has too many digits for current context')++        ans = self._rescale(exp._exp, rounding)+        if ans.adjusted() > context.Emax:+            return context._raise_error(InvalidOperation,+                                        'exponent of quantize result too large for current context')+        if len(ans._int) > context.prec:+            return context._raise_error(InvalidOperation,+                                        'quantize result has too many digits for current context')++        # raise appropriate flags+        if ans and ans.adjusted() < context.Emin:+            context._raise_error(Subnormal)+        if ans._exp > self._exp:+            if ans != self:+                context._raise_error(Inexact)+            context._raise_error(Rounded)++        # call to fix takes care of any necessary folddown, and+        # signals Clamped if necessary+        ans = ans._fix(context)+        return ans++    def same_quantum(self, other, context=None):+        """Return True if self and other have the same exponent; otherwise+        return False.++        If either operand is a special value, the following rules are used:+           * return True if both operands are infinities+           * return True if both operands are NaNs+           * otherwise, return False.+        """+        other = _convert_other(other, raiseit=True)+        if self._is_special or other._is_special:+            return (self.is_nan() and other.is_nan() or+                    self.is_infinite() and other.is_infinite())+        return self._exp == other._exp++    def _rescale(self, exp, rounding):+        """Rescale self so that the exponent is exp, either by padding with zeros+        or by truncating digits, using the given rounding mode.++        Specials are returned without change.  This operation is+        quiet: it raises no flags, and uses no information from the+        context.++        exp = exp to scale to (an integer)+        rounding = rounding mode+        """+        if self._is_special:+            return Decimal(self)+        if not self:+            return _dec_from_triple(self._sign, '0', exp)++        if self._exp >= exp:+            # pad answer with zeros if necessary+            return _dec_from_triple(self._sign,+                                        self._int + '0'*(self._exp - exp), exp)++        # too many digits; round and lose data.  If self.adjusted() <+        # exp-1, replace self by 10**(exp-1) before rounding+        digits = len(self._int) + self._exp - exp+        if digits < 0:+            self = _dec_from_triple(self._sign, '1', exp-1)+            digits = 0+        this_function = self._pick_rounding_function[rounding]+        changed = this_function(self, digits)+        coeff = self._int[:digits] or '0'+        if changed == 1:+            coeff = str(int(coeff)+1)+        return _dec_from_triple(self._sign, coeff, exp)++    def _round(self, places, rounding):+        """Round a nonzero, nonspecial Decimal to a fixed number of+        significant figures, using the given rounding mode.++        Infinities, NaNs and zeros are returned unaltered.++        This operation is quiet: it raises no flags, and uses no+        information from the context.++        """+        if places <= 0:+            raise ValueError("argument should be at least 1 in _round")+        if self._is_special or not self:+            return Decimal(self)+        ans = self._rescale(self.adjusted()+1-places, rounding)+        # it can happen that the rescale alters the adjusted exponent;+        # for example when rounding 99.97 to 3 significant figures.+        # When this happens we end up with an extra 0 at the end of+        # the number; a second rescale fixes this.+        if ans.adjusted() != self.adjusted():+            ans = ans._rescale(ans.adjusted()+1-places, rounding)+        return ans++    def to_integral_exact(self, rounding=None, context=None):+        """Rounds to a nearby integer.++        If no rounding mode is specified, take the rounding mode from+        the context.  This method raises the Rounded and Inexact flags+        when appropriate.++        See also: to_integral_value, which does exactly the same as+        this method except that it doesn't raise Inexact or Rounded.+        """+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans+            return Decimal(self)+        if self._exp >= 0:+            return Decimal(self)+        if not self:+            return _dec_from_triple(self._sign, '0', 0)+        if context is None:+            context = getcontext()+        if rounding is None:+            rounding = context.rounding+        ans = self._rescale(0, rounding)+        if ans != self:+            context._raise_error(Inexact)+        context._raise_error(Rounded)+        return ans++    def to_integral_value(self, rounding=None, context=None):+        """Rounds to the nearest integer, without raising inexact, rounded."""+        if context is None:+            context = getcontext()+        if rounding is None:+            rounding = context.rounding+        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans+            return Decimal(self)+        if self._exp >= 0:+            return Decimal(self)+        else:+            return self._rescale(0, rounding)++    # the method name changed, but we provide also the old one, for compatibility+    to_integral = to_integral_value++    def sqrt(self, context=None):+        """Return the square root of self."""+        if context is None:+            context = getcontext()++        if self._is_special:+            ans = self._check_nans(context=context)+            if ans:+                return ans++            if self._isinfinity() and self._sign == 0:+                return Decimal(self)++        if not self:+            # exponent = self._exp // 2.  sqrt(-0) = -0+            ans = _dec_from_triple(self._sign, '0', self._exp // 2)+            return ans._fix(context)++        if self._sign == 1:+            return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0')++        # At this point self represents a positive number.  Let p be+        # the desired precision and express self in the form c*100**e+        # with c a positive real number and e an integer, c and e+        # being chosen so that 100**(p-1) <= c < 100**p.  Then the+        # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1)+        # <= sqrt(c) < 10**p, so the closest representable Decimal at+        # precision p is n*10**e where n = round_half_even(sqrt(c)),+        # the closest integer to sqrt(c) with the even integer chosen+        # in the case of a tie.+        #+        # To ensure correct rounding in all cases, we use the+        # following trick: we compute the square root to an extra+        # place (precision p+1 instead of precision p), rounding down.+        # Then, if the result is inexact and its last digit is 0 or 5,+        # we increase the last digit to 1 or 6 respectively; if it's+        # exact we leave the last digit alone.  Now the final round to+        # p places (or fewer in the case of underflow) will round+        # correctly and raise the appropriate flags.++        # use an extra digit of precision+        prec = context.prec+1++        # write argument in the form c*100**e where e = self._exp//2+        # is the 'ideal' exponent, to be used if the square root is+        # exactly representable.  l is the number of 'digits' of c in+        # base 100, so that 100**(l-1) <= c < 100**l.+        op = _WorkRep(self)+        e = op.exp >> 1+        if op.exp & 1:+            c = op.int * 10+            l = (len(self._int) >> 1) + 1+        else:+            c = op.int+            l = len(self._int)+1 >> 1++        # rescale so that c has exactly prec base 100 'digits'+        shift = prec-l+        if shift >= 0:+            c *= 100**shift+            exact = True+        else:+            c, remainder = divmod(c, 100**-shift)+            exact = not remainder+        e -= shift++        # find n = floor(sqrt(c)) using Newton's method+        n = 10**prec+        while True:+            q = c//n+            if n <= q:+                break+            else:+                n = n + q >> 1+        exact = exact and n*n == c++        if exact:+            # result is exact; rescale to use ideal exponent e+            if shift >= 0:+                # assert n % 10**shift == 0+                n //= 10**shift+            else:+                n *= 10**-shift+            e += shift+        else:+            # result is not exact; fix last digit as described above+            if n % 5 == 0:+                n += 1++        ans = _dec_from_triple(0, str(n), e)++        # round, and fit to current context+        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding++        return ans++    def max(self, other, context=None):+        """Returns the larger value.++        Like max(self, other) except if one is not a number, returns+        NaN (and signals if one is sNaN).  Also rounds.+        """+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self._cmp(other)+        if c == 0:+            # If both operands are finite and equal in numerical value+            # then an ordering is applied:+            #+            # If the signs differ then max returns the operand with the+            # positive sign and min returns the operand with the negative sign+            #+            # If the signs are the same then the exponent is used to select+            # the result.  This is exactly the ordering used in compare_total.+            c = self.compare_total(other)++        if c == -1:+            ans = other+        else:+            ans = self++        return ans._fix(context)++    def min(self, other, context=None):+        """Returns the smaller value.++        Like min(self, other) except if one is not a number, returns+        NaN (and signals if one is sNaN).  Also rounds.+        """+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self._cmp(other)+        if c == 0:+            c = self.compare_total(other)++        if c == -1:+            ans = self+        else:+            ans = other++        return ans._fix(context)++    def _isinteger(self):+        """Returns whether self is an integer"""+        if self._is_special:+            return False+        if self._exp >= 0:+            return True+        rest = self._int[self._exp:]+        return rest == '0'*len(rest)++    def _iseven(self):+        """Returns True if self is even.  Assumes self is an integer."""+        if not self or self._exp > 0:+            return True+        return self._int[-1+self._exp] in '02468'++    def adjusted(self):+        """Return the adjusted exponent of self"""+        try:+            return self._exp + len(self._int) - 1+        # If NaN or Infinity, self._exp is string+        except TypeError:+            return 0++    def canonical(self):+        """Returns the same Decimal object.++        As we do not have different encodings for the same number, the+        received object already is in its canonical form.+        """+        return self++    def compare_signal(self, other, context=None):+        """Compares self to the other operand numerically.++        It's pretty much like compare(), but all NaNs signal, with signaling+        NaNs taking precedence over quiet NaNs.+        """+        other = _convert_other(other, raiseit = True)+        ans = self._compare_check_nans(other, context)+        if ans:+            return ans+        return self.compare(other, context=context)++    def compare_total(self, other, context=None):+        """Compares self to other using the abstract representations.++        This is not like the standard compare, which use their numerical+        value. Note that a total ordering is defined for all possible abstract+        representations.+        """+        other = _convert_other(other, raiseit=True)++        # if one is negative and the other is positive, it's easy+        if self._sign and not other._sign:+            return _NegativeOne+        if not self._sign and other._sign:+            return _One+        sign = self._sign++        # let's handle both NaN types+        self_nan = self._isnan()+        other_nan = other._isnan()+        if self_nan or other_nan:+            if self_nan == other_nan:+                # compare payloads as though they're integers+                self_key = len(self._int), self._int+                other_key = len(other._int), other._int+                if self_key < other_key:+                    if sign:+                        return _One+                    else:+                        return _NegativeOne+                if self_key > other_key:+                    if sign:+                        return _NegativeOne+                    else:+                        return _One+                return _Zero++            if sign:+                if self_nan == 1:+                    return _NegativeOne+                if other_nan == 1:+                    return _One+                if self_nan == 2:+                    return _NegativeOne+                if other_nan == 2:+                    return _One+            else:+                if self_nan == 1:+                    return _One+                if other_nan == 1:+                    return _NegativeOne+                if self_nan == 2:+                    return _One+                if other_nan == 2:+                    return _NegativeOne++        if self < other:+            return _NegativeOne+        if self > other:+            return _One++        if self._exp < other._exp:+            if sign:+                return _One+            else:+                return _NegativeOne+        if self._exp > other._exp:+            if sign:+                return _NegativeOne+            else:+                return _One+        return _Zero+++    def compare_total_mag(self, other, context=None):+        """Compares self to other using abstract repr., ignoring sign.++        Like compare_total, but with operand's sign ignored and assumed to be 0.+        """+        other = _convert_other(other, raiseit=True)++        s = self.copy_abs()+        o = other.copy_abs()+        return s.compare_total(o)++    def copy_abs(self):+        """Returns a copy with the sign set to 0. """+        return _dec_from_triple(0, self._int, self._exp, self._is_special)++    def copy_negate(self):+        """Returns a copy with the sign inverted."""+        if self._sign:+            return _dec_from_triple(0, self._int, self._exp, self._is_special)+        else:+            return _dec_from_triple(1, self._int, self._exp, self._is_special)++    def copy_sign(self, other, context=None):+        """Returns self with the sign of other."""+        other = _convert_other(other, raiseit=True)+        return _dec_from_triple(other._sign, self._int,+                                self._exp, self._is_special)++    def exp(self, context=None):+        """Returns e ** self."""++        if context is None:+            context = getcontext()++        # exp(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        # exp(-Infinity) = 0+        if self._isinfinity() == -1:+            return _Zero++        # exp(0) = 1+        if not self:+            return _One++        # exp(Infinity) = Infinity+        if self._isinfinity() == 1:+            return Decimal(self)++        # the result is now guaranteed to be inexact (the true+        # mathematical result is transcendental). There's no need to+        # raise Rounded and Inexact here---they'll always be raised as+        # a result of the call to _fix.+        p = context.prec+        adj = self.adjusted()++        # we only need to do any computation for quite a small range+        # of adjusted exponents---for example, -29 <= adj <= 10 for+        # the default context.  For smaller exponent the result is+        # indistinguishable from 1 at the given precision, while for+        # larger exponent the result either overflows or underflows.+        if self._sign == 0 and adj > len(str((context.Emax+1)*3)):+            # overflow+            ans = _dec_from_triple(0, '1', context.Emax+1)+        elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)):+            # underflow to 0+            ans = _dec_from_triple(0, '1', context.Etiny()-1)+        elif self._sign == 0 and adj < -p:+            # p+1 digits; final round will raise correct flags+            ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p)+        elif self._sign == 1 and adj < -p-1:+            # p+1 digits; final round will raise correct flags+            ans = _dec_from_triple(0, '9'*(p+1), -p-1)+        # general case+        else:+            op = _WorkRep(self)+            c, e = op.int, op.exp+            if op.sign == 1:+                c = -c++            # compute correctly rounded result: increase precision by+            # 3 digits at a time until we get an unambiguously+            # roundable result+            extra = 3+            while True:+                coeff, exp = _dexp(c, e, p+extra)+                if coeff % (5*10**(len(str(coeff))-p-1)):+                    break+                extra += 3++            ans = _dec_from_triple(0, str(coeff), exp)++        # at this stage, ans should round correctly with *any*+        # rounding mode, not just with ROUND_HALF_EVEN+        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding++        return ans++    def is_canonical(self):+        """Return True if self is canonical; otherwise return False.++        Currently, the encoding of a Decimal instance is always+        canonical, so this method returns True for any Decimal.+        """+        return True++    def is_finite(self):+        """Return True if self is finite; otherwise return False.++        A Decimal instance is considered finite if it is neither+        infinite nor a NaN.+        """+        return not self._is_special++    def is_infinite(self):+        """Return True if self is infinite; otherwise return False."""+        return self._exp == 'F'++    def is_nan(self):+        """Return True if self is a qNaN or sNaN; otherwise return False."""+        return self._exp in ('n', 'N')++    def is_normal(self, context=None):+        """Return True if self is a normal number; otherwise return False."""+        if self._is_special or not self:+            return False+        if context is None:+            context = getcontext()+        return context.Emin <= self.adjusted()++    def is_qnan(self):+        """Return True if self is a quiet NaN; otherwise return False."""+        return self._exp == 'n'++    def is_signed(self):+        """Return True if self is negative; otherwise return False."""+        return self._sign == 1++    def is_snan(self):+        """Return True if self is a signaling NaN; otherwise return False."""+        return self._exp == 'N'++    def is_subnormal(self, context=None):+        """Return True if self is subnormal; otherwise return False."""+        if self._is_special or not self:+            return False+        if context is None:+            context = getcontext()+        return self.adjusted() < context.Emin++    def is_zero(self):+        """Return True if self is a zero; otherwise return False."""+        return not self._is_special and self._int == '0'++    def _ln_exp_bound(self):+        """Compute a lower bound for the adjusted exponent of self.ln().+        In other words, compute r such that self.ln() >= 10**r.  Assumes+        that self is finite and positive and that self != 1.+        """++        # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1+        adj = self._exp + len(self._int) - 1+        if adj >= 1:+            # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10)+            return len(str(adj*23//10)) - 1+        if adj <= -2:+            # argument <= 0.1+            return len(str((-1-adj)*23//10)) - 1+        op = _WorkRep(self)+        c, e = op.int, op.exp+        if adj == 0:+            # 1 < self < 10+            num = str(c-10**-e)+            den = str(c)+            return len(num) - len(den) - (num < den)+        # adj == -1, 0.1 <= self < 1+        return e + len(str(10**-e - c)) - 1+++    def ln(self, context=None):+        """Returns the natural (base e) logarithm of self."""++        if context is None:+            context = getcontext()++        # ln(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        # ln(0.0) == -Infinity+        if not self:+            return _NegativeInfinity++        # ln(Infinity) = Infinity+        if self._isinfinity() == 1:+            return _Infinity++        # ln(1.0) == 0.0+        if self == _One:+            return _Zero++        # ln(negative) raises InvalidOperation+        if self._sign == 1:+            return context._raise_error(InvalidOperation,+                                        'ln of a negative value')++        # result is irrational, so necessarily inexact+        op = _WorkRep(self)+        c, e = op.int, op.exp+        p = context.prec++        # correctly rounded result: repeatedly increase precision by 3+        # until we get an unambiguously roundable result+        places = p - self._ln_exp_bound() + 2 # at least p+3 places+        while True:+            coeff = _dlog(c, e, places)+            # assert len(str(abs(coeff)))-p >= 1+            if coeff % (5*10**(len(str(abs(coeff)))-p-1)):+                break+            places += 3+        ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)++        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding+        return ans++    def _log10_exp_bound(self):+        """Compute a lower bound for the adjusted exponent of self.log10().+        In other words, find r such that self.log10() >= 10**r.+        Assumes that self is finite and positive and that self != 1.+        """++        # For x >= 10 or x < 0.1 we only need a bound on the integer+        # part of log10(self), and this comes directly from the+        # exponent of x.  For 0.1 <= x <= 10 we use the inequalities+        # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| >+        # (1-1/x)/2.31 > 0.  If x < 1 then |log10(x)| > (1-x)/2.31 > 0++        adj = self._exp + len(self._int) - 1+        if adj >= 1:+            # self >= 10+            return len(str(adj))-1+        if adj <= -2:+            # self < 0.1+            return len(str(-1-adj))-1+        op = _WorkRep(self)+        c, e = op.int, op.exp+        if adj == 0:+            # 1 < self < 10+            num = str(c-10**-e)+            den = str(231*c)+            return len(num) - len(den) - (num < den) + 2+        # adj == -1, 0.1 <= self < 1+        num = str(10**-e-c)+        return len(num) + e - (num < "231") - 1++    def log10(self, context=None):+        """Returns the base 10 logarithm of self."""++        if context is None:+            context = getcontext()++        # log10(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        # log10(0.0) == -Infinity+        if not self:+            return _NegativeInfinity++        # log10(Infinity) = Infinity+        if self._isinfinity() == 1:+            return _Infinity++        # log10(negative or -Infinity) raises InvalidOperation+        if self._sign == 1:+            return context._raise_error(InvalidOperation,+                                        'log10 of a negative value')++        # log10(10**n) = n+        if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1):+            # answer may need rounding+            ans = Decimal(self._exp + len(self._int) - 1)+        else:+            # result is irrational, so necessarily inexact+            op = _WorkRep(self)+            c, e = op.int, op.exp+            p = context.prec++            # correctly rounded result: repeatedly increase precision+            # until result is unambiguously roundable+            places = p-self._log10_exp_bound()+2+            while True:+                coeff = _dlog10(c, e, places)+                # assert len(str(abs(coeff)))-p >= 1+                if coeff % (5*10**(len(str(abs(coeff)))-p-1)):+                    break+                places += 3+            ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)++        context = context._shallow_copy()+        rounding = context._set_rounding(ROUND_HALF_EVEN)+        ans = ans._fix(context)+        context.rounding = rounding+        return ans++    def logb(self, context=None):+        """ Returns the exponent of the magnitude of self's MSD.++        The result is the integer which is the exponent of the magnitude+        of the most significant digit of self (as though it were truncated+        to a single digit while maintaining the value of that digit and+        without limiting the resulting exponent).+        """+        # logb(NaN) = NaN+        ans = self._check_nans(context=context)+        if ans:+            return ans++        if context is None:+            context = getcontext()++        # logb(+/-Inf) = +Inf+        if self._isinfinity():+            return _Infinity++        # logb(0) = -Inf, DivisionByZero+        if not self:+            return context._raise_error(DivisionByZero, 'logb(0)', 1)++        # otherwise, simply return the adjusted exponent of self, as a+        # Decimal.  Note that no attempt is made to fit the result+        # into the current context.+        ans = Decimal(self.adjusted())+        return ans._fix(context)++    def _islogical(self):+        """Return True if self is a logical operand.++        For being logical, it must be a finite number with a sign of 0,+        an exponent of 0, and a coefficient whose digits must all be+        either 0 or 1.+        """+        if self._sign != 0 or self._exp != 0:+            return False+        for dig in self._int:+            if dig not in '01':+                return False+        return True++    def _fill_logical(self, context, opa, opb):+        dif = context.prec - len(opa)+        if dif > 0:+            opa = '0'*dif + opa+        elif dif < 0:+            opa = opa[-context.prec:]+        dif = context.prec - len(opb)+        if dif > 0:+            opb = '0'*dif + opb+        elif dif < 0:+            opb = opb[-context.prec:]+        return opa, opb++    def logical_and(self, other, context=None):+        """Applies an 'and' operation between self and other's digits."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        if not self._islogical() or not other._islogical():+            return context._raise_error(InvalidOperation)++        # fill to context.prec+        (opa, opb) = self._fill_logical(context, self._int, other._int)++        # make the operation, and clean starting zeroes+        result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)])+        return _dec_from_triple(0, result.lstrip('0') or '0', 0)++    def logical_invert(self, context=None):+        """Invert all its digits."""+        if context is None:+            context = getcontext()+        return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0),+                                context)++    def logical_or(self, other, context=None):+        """Applies an 'or' operation between self and other's digits."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        if not self._islogical() or not other._islogical():+            return context._raise_error(InvalidOperation)++        # fill to context.prec+        (opa, opb) = self._fill_logical(context, self._int, other._int)++        # make the operation, and clean starting zeroes+        result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)])+        return _dec_from_triple(0, result.lstrip('0') or '0', 0)++    def logical_xor(self, other, context=None):+        """Applies an 'xor' operation between self and other's digits."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        if not self._islogical() or not other._islogical():+            return context._raise_error(InvalidOperation)++        # fill to context.prec+        (opa, opb) = self._fill_logical(context, self._int, other._int)++        # make the operation, and clean starting zeroes+        result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)])+        return _dec_from_triple(0, result.lstrip('0') or '0', 0)++    def max_mag(self, other, context=None):+        """Compares the values numerically with their sign ignored."""+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self.copy_abs()._cmp(other.copy_abs())+        if c == 0:+            c = self.compare_total(other)++        if c == -1:+            ans = other+        else:+            ans = self++        return ans._fix(context)++    def min_mag(self, other, context=None):+        """Compares the values numerically with their sign ignored."""+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        if self._is_special or other._is_special:+            # If one operand is a quiet NaN and the other is number, then the+            # number is always returned+            sn = self._isnan()+            on = other._isnan()+            if sn or on:+                if on == 1 and sn == 0:+                    return self._fix(context)+                if sn == 1 and on == 0:+                    return other._fix(context)+                return self._check_nans(other, context)++        c = self.copy_abs()._cmp(other.copy_abs())+        if c == 0:+            c = self.compare_total(other)++        if c == -1:+            ans = self+        else:+            ans = other++        return ans._fix(context)++    def next_minus(self, context=None):+        """Returns the largest representable number smaller than itself."""+        if context is None:+            context = getcontext()++        ans = self._check_nans(context=context)+        if ans:+            return ans++        if self._isinfinity() == -1:+            return _NegativeInfinity+        if self._isinfinity() == 1:+            return _dec_from_triple(0, '9'*context.prec, context.Etop())++        context = context.copy()+        context._set_rounding(ROUND_FLOOR)+        context._ignore_all_flags()+        new_self = self._fix(context)+        if new_self != self:+            return new_self+        return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1),+                            context)++    def next_plus(self, context=None):+        """Returns the smallest representable number larger than itself."""+        if context is None:+            context = getcontext()++        ans = self._check_nans(context=context)+        if ans:+            return ans++        if self._isinfinity() == 1:+            return _Infinity+        if self._isinfinity() == -1:+            return _dec_from_triple(1, '9'*context.prec, context.Etop())++        context = context.copy()+        context._set_rounding(ROUND_CEILING)+        context._ignore_all_flags()+        new_self = self._fix(context)+        if new_self != self:+            return new_self+        return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1),+                            context)++    def next_toward(self, other, context=None):+        """Returns the number closest to self, in the direction towards other.++        The result is the closest representable number to self+        (excluding self) that is in the direction towards other,+        unless both have the same value.  If the two operands are+        numerically equal, then the result is a copy of self with the+        sign set to be the same as the sign of other.+        """+        other = _convert_other(other, raiseit=True)++        if context is None:+            context = getcontext()++        ans = self._check_nans(other, context)+        if ans:+            return ans++        comparison = self._cmp(other)+        if comparison == 0:+            return self.copy_sign(other)++        if comparison == -1:+            ans = self.next_plus(context)+        else: # comparison == 1+            ans = self.next_minus(context)++        # decide which flags to raise using value of ans+        if ans._isinfinity():+            context._raise_error(Overflow,+                                 'Infinite result from next_toward',+                                 ans._sign)+            context._raise_error(Inexact)+            context._raise_error(Rounded)+        elif ans.adjusted() < context.Emin:+            context._raise_error(Underflow)+            context._raise_error(Subnormal)+            context._raise_error(Inexact)+            context._raise_error(Rounded)+            # if precision == 1 then we don't raise Clamped for a+            # result 0E-Etiny.+            if not ans:+                context._raise_error(Clamped)++        return ans++    def number_class(self, context=None):+        """Returns an indication of the class of self.++        The class is one of the following strings:+          sNaN+          NaN+          -Infinity+          -Normal+          -Subnormal+          -Zero+          +Zero+          +Subnormal+          +Normal+          +Infinity+        """+        if self.is_snan():+            return "sNaN"+        if self.is_qnan():+            return "NaN"+        inf = self._isinfinity()+        if inf == 1:+            return "+Infinity"+        if inf == -1:+            return "-Infinity"+        if self.is_zero():+            if self._sign:+                return "-Zero"+            else:+                return "+Zero"+        if context is None:+            context = getcontext()+        if self.is_subnormal(context=context):+            if self._sign:+                return "-Subnormal"+            else:+                return "+Subnormal"+        # just a normal, regular, boring number, :)+        if self._sign:+            return "-Normal"+        else:+            return "+Normal"++    def radix(self):+        """Just returns 10, as this is Decimal, :)"""+        return Decimal(10)++    def rotate(self, other, context=None):+        """Returns a rotated copy of self, value-of-other times."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if other._exp != 0:+            return context._raise_error(InvalidOperation)+        if not (-context.prec <= int(other) <= context.prec):+            return context._raise_error(InvalidOperation)++        if self._isinfinity():+            return Decimal(self)++        # get values, pad if necessary+        torot = int(other)+        rotdig = self._int+        topad = context.prec - len(rotdig)+        if topad > 0:+            rotdig = '0'*topad + rotdig+        elif topad < 0:+            rotdig = rotdig[-topad:]++        # let's rotate!+        rotated = rotdig[torot:] + rotdig[:torot]+        return _dec_from_triple(self._sign,+                                rotated.lstrip('0') or '0', self._exp)++    def scaleb(self, other, context=None):+        """Returns self operand after adding the second value to its exp."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if other._exp != 0:+            return context._raise_error(InvalidOperation)+        liminf = -2 * (context.Emax + context.prec)+        limsup =  2 * (context.Emax + context.prec)+        if not (liminf <= int(other) <= limsup):+            return context._raise_error(InvalidOperation)++        if self._isinfinity():+            return Decimal(self)++        d = _dec_from_triple(self._sign, self._int, self._exp + int(other))+        d = d._fix(context)+        return d++    def shift(self, other, context=None):+        """Returns a shifted copy of self, value-of-other times."""+        if context is None:+            context = getcontext()++        other = _convert_other(other, raiseit=True)++        ans = self._check_nans(other, context)+        if ans:+            return ans++        if other._exp != 0:+            return context._raise_error(InvalidOperation)+        if not (-context.prec <= int(other) <= context.prec):+            return context._raise_error(InvalidOperation)++        if self._isinfinity():+            return Decimal(self)++        # get values, pad if necessary+        torot = int(other)+        rotdig = self._int+        topad = context.prec - len(rotdig)+        if topad > 0:+            rotdig = '0'*topad + rotdig+        elif topad < 0:+            rotdig = rotdig[-topad:]++        # let's shift!+        if torot < 0:+            shifted = rotdig[:torot]+        else:+            shifted = rotdig + '0'*torot+            shifted = shifted[-context.prec:]++        return _dec_from_triple(self._sign,+                                    shifted.lstrip('0') or '0', self._exp)++    # Support for pickling, copy, and deepcopy+    def __reduce__(self):+        return (self.__class__, (str(self),))++    def __copy__(self):+        if type(self) is Decimal:+            return self     # I'm immutable; therefore I am my own clone+        return self.__class__(str(self))++    def __deepcopy__(self, memo):+        if type(self) is Decimal:+            return self     # My components are also immutable+        return self.__class__(str(self))++    # PEP 3101 support.  the _localeconv keyword argument should be+    # considered private: it's provided for ease of testing only.+    def __format__(self, specifier, context=None, _localeconv=None):+        """Format a Decimal instance according to the given specifier.++        The specifier should be a standard format specifier, with the+        form described in PEP 3101.  Formatting types 'e', 'E', 'f',+        'F', 'g', 'G', 'n' and '%' are supported.  If the formatting+        type is omitted it defaults to 'g' or 'G', depending on the+        value of context.capitals.+        """++        # Note: PEP 3101 says that if the type is not present then+        # there should be at least one digit after the decimal point.+        # We take the liberty of ignoring this requirement for+        # Decimal---it's presumably there to make sure that+        # format(float, '') behaves similarly to str(float).+        if context is None:+            context = getcontext()++        spec = _parse_format_specifier(specifier, _localeconv=_localeconv)++        # special values don't care about the type or precision+        if self._is_special:+            sign = _format_sign(self._sign, spec)+            body = str(self.copy_abs())+            if spec['type'] == '%':+                body += '%'+            return _format_align(sign, body, spec)++        # a type of None defaults to 'g' or 'G', depending on context+        if spec['type'] is None:+            spec['type'] = ['g', 'G'][context.capitals]++        # if type is '%', adjust exponent of self accordingly+        if spec['type'] == '%':+            self = _dec_from_triple(self._sign, self._int, self._exp+2)++        # round if necessary, taking rounding mode from the context+        rounding = context.rounding+        precision = spec['precision']+        if precision is not None:+            if spec['type'] in 'eE':+                self = self._round(precision+1, rounding)+            elif spec['type'] in 'fF%':+                self = self._rescale(-precision, rounding)+            elif spec['type'] in 'gG' and len(self._int) > precision:+                self = self._round(precision, rounding)+        # special case: zeros with a positive exponent can't be+        # represented in fixed point; rescale them to 0e0.+        if not self and self._exp > 0 and spec['type'] in 'fF%':+            self = self._rescale(0, rounding)++        # figure out placement of the decimal point+        leftdigits = self._exp + len(self._int)+        if spec['type'] in 'eE':+            if not self and precision is not None:+                dotplace = 1 - precision+            else:+                dotplace = 1+        elif spec['type'] in 'fF%':+            dotplace = leftdigits+        elif spec['type'] in 'gG':+            if self._exp <= 0 and leftdigits > -6:+                dotplace = leftdigits+            else:+                dotplace = 1++        # find digits before and after decimal point, and get exponent+        if dotplace < 0:+            intpart = '0'+            fracpart = '0'*(-dotplace) + self._int+        elif dotplace > len(self._int):+            intpart = self._int + '0'*(dotplace-len(self._int))+            fracpart = ''+        else:+            intpart = self._int[:dotplace] or '0'+            fracpart = self._int[dotplace:]+        exp = leftdigits-dotplace++        # done with the decimal-specific stuff;  hand over the rest+        # of the formatting to the _format_number function+        return _format_number(self._sign, intpart, fracpart, exp, spec)++def _dec_from_triple(sign, coefficient, exponent, special=False):+    """Create a decimal instance directly, without any validation,+    normalization (e.g. removal of leading zeros) or argument+    conversion.++    This function is for *internal use only*.+    """++    self = object.__new__(Decimal)+    self._sign = sign+    self._int = coefficient+    self._exp = exponent+    self._is_special = special++    return self++# Register Decimal as a kind of Number (an abstract base class).+# However, do not register it as Real (because Decimals are not+# interoperable with floats).+_numbers.Number.register(Decimal)+++##### Context class #######################################################++class _ContextManager(object):+    """Context manager class to support localcontext().++      Sets a copy of the supplied context in __enter__() and restores+      the previous decimal context in __exit__()+    """+    def __init__(self, new_context):+        self.new_context = new_context.copy()+    def __enter__(self):+        self.saved_context = getcontext()+        setcontext(self.new_context)+        return self.new_context+    def __exit__(self, t, v, tb):+        setcontext(self.saved_context)++class Context(object):+    """Contains the context for a Decimal instance.++    Contains:+    prec - precision (for use in rounding, division, square roots..)+    rounding - rounding type (how you round)+    traps - If traps[exception] = 1, then the exception is+                    raised when it is caused.  Otherwise, a value is+                    substituted in.+    flags  - When an exception is caused, flags[exception] is set.+             (Whether or not the trap_enabler is set)+             Should be reset by user of Decimal instance.+    Emin -   Minimum exponent+    Emax -   Maximum exponent+    capitals -      If 1, 1*10^1 is printed as 1E+1.+                    If 0, printed as 1e1+    clamp -  If 1, change exponents if too high (Default 0)+    """++    def __init__(self, prec=None, rounding=None, Emin=None, Emax=None,+                       capitals=None, clamp=None, flags=None, traps=None,+                       _ignored_flags=None):+        # Set defaults; for everything except flags and _ignored_flags,+        # inherit from DefaultContext.+        try:+            dc = DefaultContext+        except NameError:+            pass++        self.prec = prec if prec is not None else dc.prec+        self.rounding = rounding if rounding is not None else dc.rounding+        self.Emin = Emin if Emin is not None else dc.Emin+        self.Emax = Emax if Emax is not None else dc.Emax+        self.capitals = capitals if capitals is not None else dc.capitals+        self.clamp = clamp if clamp is not None else dc.clamp++        if _ignored_flags is None:+            self._ignored_flags = []+        else:+            self._ignored_flags = _ignored_flags++        if traps is None:+            self.traps = dc.traps.copy()+        elif not isinstance(traps, dict):+            self.traps = dict((s, int(s in traps)) for s in _signals + traps)+        else:+            self.traps = traps++        if flags is None:+            self.flags = dict.fromkeys(_signals, 0)+        elif not isinstance(flags, dict):+            self.flags = dict((s, int(s in flags)) for s in _signals + flags)+        else:+            self.flags = flags++    def _set_integer_check(self, name, value, vmin, vmax):+        if not isinstance(value, int):+            raise TypeError("%s must be an integer" % name)+        if vmin == '-inf':+            if value > vmax:+                raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value))+        elif vmax == 'inf':+            if value < vmin:+                raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value))+        else:+            if value < vmin or value > vmax:+                raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value))+        return object.__setattr__(self, name, value)++    def _set_signal_dict(self, name, d):+        if not isinstance(d, dict):+            raise TypeError("%s must be a signal dict" % d)+        for key in d:+            if not key in _signals:+                raise KeyError("%s is not a valid signal dict" % d)+        for key in _signals:+            if not key in d:+                raise KeyError("%s is not a valid signal dict" % d)+        return object.__setattr__(self, name, d)++    def __setattr__(self, name, value):+        if name == 'prec':+            return self._set_integer_check(name, value, 1, 'inf')+        elif name == 'Emin':+            return self._set_integer_check(name, value, '-inf', 0)+        elif name == 'Emax':+            return self._set_integer_check(name, value, 0, 'inf')+        elif name == 'capitals':+            return self._set_integer_check(name, value, 0, 1)+        elif name == 'clamp':+            return self._set_integer_check(name, value, 0, 1)+        elif name == 'rounding':+            if not value in _rounding_modes:+                # raise TypeError even for strings to have consistency+                # among various implementations.+                raise TypeError("%s: invalid rounding mode" % value)+            return object.__setattr__(self, name, value)+        elif name == 'flags' or name == 'traps':+            return self._set_signal_dict(name, value)+        elif name == '_ignored_flags':+            return object.__setattr__(self, name, value)+        else:+            raise AttributeError(+                "'decimal.Context' object has no attribute '%s'" % name)++    def __delattr__(self, name):+        raise AttributeError("%s cannot be deleted" % name)++    # Support for pickling, copy, and deepcopy+    def __reduce__(self):+        flags = [sig for sig, v in self.flags.items() if v]+        traps = [sig for sig, v in self.traps.items() if v]+        return (self.__class__,+                (self.prec, self.rounding, self.Emin, self.Emax,+                 self.capitals, self.clamp, flags, traps))++    def __repr__(self):+        """Show the current context."""+        s = []+        s.append('Context(prec=%(prec)d, rounding=%(rounding)s, '+                 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, '+                 'clamp=%(clamp)d'+                 % vars(self))+        names = [f.__name__ for f, v in self.flags.items() if v]+        s.append('flags=[' + ', '.join(names) + ']')+        names = [t.__name__ for t, v in self.traps.items() if v]+        s.append('traps=[' + ', '.join(names) + ']')+        return ', '.join(s) + ')'++    def clear_flags(self):+        """Reset all flags to zero"""+        for flag in self.flags:+            self.flags[flag] = 0++    def clear_traps(self):+        """Reset all traps to zero"""+        for flag in self.traps:+            self.traps[flag] = 0++    def _shallow_copy(self):+        """Returns a shallow copy from self."""+        nc = Context(self.prec, self.rounding, self.Emin, self.Emax,+                     self.capitals, self.clamp, self.flags, self.traps,+                     self._ignored_flags)+        return nc++    def copy(self):+        """Returns a deep copy from self."""+        nc = Context(self.prec, self.rounding, self.Emin, self.Emax,+                     self.capitals, self.clamp,+                     self.flags.copy(), self.traps.copy(),+                     self._ignored_flags)+        return nc+    __copy__ = copy++    def _raise_error(self, condition, explanation = None, *args):+        """Handles an error++        If the flag is in _ignored_flags, returns the default response.+        Otherwise, it sets the flag, then, if the corresponding+        trap_enabler is set, it reraises the exception.  Otherwise, it returns+        the default value after setting the flag.+        """+        error = _condition_map.get(condition, condition)+        if error in self._ignored_flags:+            # Don't touch the flag+            return error().handle(self, *args)++        self.flags[error] = 1+        if not self.traps[error]:+            # The errors define how to handle themselves.+            return condition().handle(self, *args)++        # Errors should only be risked on copies of the context+        # self._ignored_flags = []+        raise error(explanation)++    def _ignore_all_flags(self):+        """Ignore all flags, if they are raised"""+        return self._ignore_flags(*_signals)++    def _ignore_flags(self, *flags):+        """Ignore the flags, if they are raised"""+        # Do not mutate-- This way, copies of a context leave the original+        # alone.+        self._ignored_flags = (self._ignored_flags + list(flags))+        return list(flags)++    def _regard_flags(self, *flags):+        """Stop ignoring the flags, if they are raised"""+        if flags and isinstance(flags[0], (tuple,list)):+            flags = flags[0]+        for flag in flags:+            self._ignored_flags.remove(flag)++    # We inherit object.__hash__, so we must deny this explicitly+    __hash__ = None++    def Etiny(self):+        """Returns Etiny (= Emin - prec + 1)"""+        return int(self.Emin - self.prec + 1)++    def Etop(self):+        """Returns maximum exponent (= Emax - prec + 1)"""+        return int(self.Emax - self.prec + 1)++    def _set_rounding(self, type):+        """Sets the rounding type.++        Sets the rounding type, and returns the current (previous)+        rounding type.  Often used like:++        context = context.copy()+        # so you don't change the calling context+        # if an error occurs in the middle.+        rounding = context._set_rounding(ROUND_UP)+        val = self.__sub__(other, context=context)+        context._set_rounding(rounding)++        This will make it round up for that operation.+        """+        rounding = self.rounding+        self.rounding= type+        return rounding++    def create_decimal(self, num='0'):+        """Creates a new Decimal instance but using self as context.++        This method implements the to-number operation of the+        IBM Decimal specification."""++        if isinstance(num, str) and num != num.strip():+            return self._raise_error(ConversionSyntax,+                                     "no trailing or leading whitespace is "+                                     "permitted.")++        d = Decimal(num, context=self)+        if d._isnan() and len(d._int) > self.prec - self.clamp:+            return self._raise_error(ConversionSyntax,+                                     "diagnostic info too long in NaN")+        return d._fix(self)++    def create_decimal_from_float(self, f):+        """Creates a new Decimal instance from a float but rounding using self+        as the context.++        >>> context = Context(prec=5, rounding=ROUND_DOWN)+        >>> context.create_decimal_from_float(3.1415926535897932)+        Decimal('3.1415')+        >>> context = Context(prec=5, traps=[Inexact])+        >>> context.create_decimal_from_float(3.1415926535897932)+        Traceback (most recent call last):+            ...+        decimal.Inexact: None++        """+        d = Decimal.from_float(f)       # An exact conversion+        return d._fix(self)             # Apply the context rounding++    # Methods+    def abs(self, a):+        """Returns the absolute value of the operand.++        If the operand is negative, the result is the same as using the minus+        operation on the operand.  Otherwise, the result is the same as using+        the plus operation on the operand.++        >>> ExtendedContext.abs(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.abs(Decimal('-100'))+        Decimal('100')+        >>> ExtendedContext.abs(Decimal('101.5'))+        Decimal('101.5')+        >>> ExtendedContext.abs(Decimal('-101.5'))+        Decimal('101.5')+        >>> ExtendedContext.abs(-1)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.__abs__(context=self)++    def add(self, a, b):+        """Return the sum of the two operands.++        >>> ExtendedContext.add(Decimal('12'), Decimal('7.00'))+        Decimal('19.00')+        >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4'))+        Decimal('1.02E+4')+        >>> ExtendedContext.add(1, Decimal(2))+        Decimal('3')+        >>> ExtendedContext.add(Decimal(8), 5)+        Decimal('13')+        >>> ExtendedContext.add(5, 5)+        Decimal('10')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__add__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def _apply(self, a):+        return str(a._fix(self))++    def canonical(self, a):+        """Returns the same Decimal object.++        As we do not have different encodings for the same number, the+        received object already is in its canonical form.++        >>> ExtendedContext.canonical(Decimal('2.50'))+        Decimal('2.50')+        """+        if not isinstance(a, Decimal):+            raise TypeError("canonical requires a Decimal as an argument.")+        return a.canonical()++    def compare(self, a, b):+        """Compares values numerically.++        If the signs of the operands differ, a value representing each operand+        ('-1' if the operand is less than zero, '0' if the operand is zero or+        negative zero, or '1' if the operand is greater than zero) is used in+        place of that operand for the comparison instead of the actual+        operand.++        The comparison is then effected by subtracting the second operand from+        the first and then returning a value according to the result of the+        subtraction: '-1' if the result is less than zero, '0' if the result is+        zero or negative zero, or '1' if the result is greater than zero.++        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3'))+        Decimal('-1')+        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1'))+        Decimal('0')+        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10'))+        Decimal('0')+        >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1'))+        Decimal('1')+        >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3'))+        Decimal('1')+        >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1'))+        Decimal('-1')+        >>> ExtendedContext.compare(1, 2)+        Decimal('-1')+        >>> ExtendedContext.compare(Decimal(1), 2)+        Decimal('-1')+        >>> ExtendedContext.compare(1, Decimal(2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.compare(b, context=self)++    def compare_signal(self, a, b):+        """Compares the values of the two operands numerically.++        It's pretty much like compare(), but all NaNs signal, with signaling+        NaNs taking precedence over quiet NaNs.++        >>> c = ExtendedContext+        >>> c.compare_signal(Decimal('2.1'), Decimal('3'))+        Decimal('-1')+        >>> c.compare_signal(Decimal('2.1'), Decimal('2.1'))+        Decimal('0')+        >>> c.flags[InvalidOperation] = 0+        >>> print(c.flags[InvalidOperation])+        0+        >>> c.compare_signal(Decimal('NaN'), Decimal('2.1'))+        Decimal('NaN')+        >>> print(c.flags[InvalidOperation])+        1+        >>> c.flags[InvalidOperation] = 0+        >>> print(c.flags[InvalidOperation])+        0+        >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1'))+        Decimal('NaN')+        >>> print(c.flags[InvalidOperation])+        1+        >>> c.compare_signal(-1, 2)+        Decimal('-1')+        >>> c.compare_signal(Decimal(-1), 2)+        Decimal('-1')+        >>> c.compare_signal(-1, Decimal(2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.compare_signal(b, context=self)++    def compare_total(self, a, b):+        """Compares two operands using their abstract representation.++        This is not like the standard compare, which use their numerical+        value. Note that a total ordering is defined for all possible abstract+        representations.++        >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal('-127'),  Decimal('12'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30'))+        Decimal('0')+        >>> ExtendedContext.compare_total(Decimal('12.3'),  Decimal('12.300'))+        Decimal('1')+        >>> ExtendedContext.compare_total(Decimal('12.3'),  Decimal('NaN'))+        Decimal('-1')+        >>> ExtendedContext.compare_total(1, 2)+        Decimal('-1')+        >>> ExtendedContext.compare_total(Decimal(1), 2)+        Decimal('-1')+        >>> ExtendedContext.compare_total(1, Decimal(2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.compare_total(b)++    def compare_total_mag(self, a, b):+        """Compares two operands using their abstract representation ignoring sign.++        Like compare_total, but with operand's sign ignored and assumed to be 0.+        """+        a = _convert_other(a, raiseit=True)+        return a.compare_total_mag(b)++    def copy_abs(self, a):+        """Returns a copy of the operand with the sign set to 0.++        >>> ExtendedContext.copy_abs(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.copy_abs(Decimal('-100'))+        Decimal('100')+        >>> ExtendedContext.copy_abs(-1)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.copy_abs()++    def copy_decimal(self, a):+        """Returns a copy of the decimal object.++        >>> ExtendedContext.copy_decimal(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.copy_decimal(Decimal('-1.00'))+        Decimal('-1.00')+        >>> ExtendedContext.copy_decimal(1)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return Decimal(a)++    def copy_negate(self, a):+        """Returns a copy of the operand with the sign inverted.++        >>> ExtendedContext.copy_negate(Decimal('101.5'))+        Decimal('-101.5')+        >>> ExtendedContext.copy_negate(Decimal('-101.5'))+        Decimal('101.5')+        >>> ExtendedContext.copy_negate(1)+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.copy_negate()++    def copy_sign(self, a, b):+        """Copies the second operand's sign to the first one.++        In detail, it returns a copy of the first operand with the sign+        equal to the sign of the second operand.++        >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33'))+        Decimal('1.50')+        >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33'))+        Decimal('1.50')+        >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33'))+        Decimal('-1.50')+        >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33'))+        Decimal('-1.50')+        >>> ExtendedContext.copy_sign(1, -2)+        Decimal('-1')+        >>> ExtendedContext.copy_sign(Decimal(1), -2)+        Decimal('-1')+        >>> ExtendedContext.copy_sign(1, Decimal(-2))+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.copy_sign(b)++    def divide(self, a, b):+        """Decimal division in a specified context.++        >>> ExtendedContext.divide(Decimal('1'), Decimal('3'))+        Decimal('0.333333333')+        >>> ExtendedContext.divide(Decimal('2'), Decimal('3'))+        Decimal('0.666666667')+        >>> ExtendedContext.divide(Decimal('5'), Decimal('2'))+        Decimal('2.5')+        >>> ExtendedContext.divide(Decimal('1'), Decimal('10'))+        Decimal('0.1')+        >>> ExtendedContext.divide(Decimal('12'), Decimal('12'))+        Decimal('1')+        >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2'))+        Decimal('4.00')+        >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0'))+        Decimal('1.20')+        >>> ExtendedContext.divide(Decimal('1000'), Decimal('100'))+        Decimal('10')+        >>> ExtendedContext.divide(Decimal('1000'), Decimal('1'))+        Decimal('1000')+        >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2'))+        Decimal('1.20E+6')+        >>> ExtendedContext.divide(5, 5)+        Decimal('1')+        >>> ExtendedContext.divide(Decimal(5), 5)+        Decimal('1')+        >>> ExtendedContext.divide(5, Decimal(5))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__truediv__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def divide_int(self, a, b):+        """Divides two numbers and returns the integer part of the result.++        >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3'))+        Decimal('0')+        >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3'))+        Decimal('3')+        >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3'))+        Decimal('3')+        >>> ExtendedContext.divide_int(10, 3)+        Decimal('3')+        >>> ExtendedContext.divide_int(Decimal(10), 3)+        Decimal('3')+        >>> ExtendedContext.divide_int(10, Decimal(3))+        Decimal('3')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__floordiv__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def divmod(self, a, b):+        """Return (a // b, a % b).++        >>> ExtendedContext.divmod(Decimal(8), Decimal(3))+        (Decimal('2'), Decimal('2'))+        >>> ExtendedContext.divmod(Decimal(8), Decimal(4))+        (Decimal('2'), Decimal('0'))+        >>> ExtendedContext.divmod(8, 4)+        (Decimal('2'), Decimal('0'))+        >>> ExtendedContext.divmod(Decimal(8), 4)+        (Decimal('2'), Decimal('0'))+        >>> ExtendedContext.divmod(8, Decimal(4))+        (Decimal('2'), Decimal('0'))+        """+        a = _convert_other(a, raiseit=True)+        r = a.__divmod__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def exp(self, a):+        """Returns e ** a.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.exp(Decimal('-Infinity'))+        Decimal('0')+        >>> c.exp(Decimal('-1'))+        Decimal('0.367879441')+        >>> c.exp(Decimal('0'))+        Decimal('1')+        >>> c.exp(Decimal('1'))+        Decimal('2.71828183')+        >>> c.exp(Decimal('0.693147181'))+        Decimal('2.00000000')+        >>> c.exp(Decimal('+Infinity'))+        Decimal('Infinity')+        >>> c.exp(10)+        Decimal('22026.4658')+        """+        a =_convert_other(a, raiseit=True)+        return a.exp(context=self)++    def fma(self, a, b, c):+        """Returns a multiplied by b, plus c.++        The first two operands are multiplied together, using multiply,+        the third operand is then added to the result of that+        multiplication, using add, all with only one final rounding.++        >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7'))+        Decimal('22')+        >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7'))+        Decimal('-8')+        >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578'))+        Decimal('1.38435736E+12')+        >>> ExtendedContext.fma(1, 3, 4)+        Decimal('7')+        >>> ExtendedContext.fma(1, Decimal(3), 4)+        Decimal('7')+        >>> ExtendedContext.fma(1, 3, Decimal(4))+        Decimal('7')+        """+        a = _convert_other(a, raiseit=True)+        return a.fma(b, c, context=self)++    def is_canonical(self, a):+        """Return True if the operand is canonical; otherwise return False.++        Currently, the encoding of a Decimal instance is always+        canonical, so this method returns True for any Decimal.++        >>> ExtendedContext.is_canonical(Decimal('2.50'))+        True+        """+        if not isinstance(a, Decimal):+            raise TypeError("is_canonical requires a Decimal as an argument.")+        return a.is_canonical()++    def is_finite(self, a):+        """Return True if the operand is finite; otherwise return False.++        A Decimal instance is considered finite if it is neither+        infinite nor a NaN.++        >>> ExtendedContext.is_finite(Decimal('2.50'))+        True+        >>> ExtendedContext.is_finite(Decimal('-0.3'))+        True+        >>> ExtendedContext.is_finite(Decimal('0'))+        True+        >>> ExtendedContext.is_finite(Decimal('Inf'))+        False+        >>> ExtendedContext.is_finite(Decimal('NaN'))+        False+        >>> ExtendedContext.is_finite(1)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_finite()++    def is_infinite(self, a):+        """Return True if the operand is infinite; otherwise return False.++        >>> ExtendedContext.is_infinite(Decimal('2.50'))+        False+        >>> ExtendedContext.is_infinite(Decimal('-Inf'))+        True+        >>> ExtendedContext.is_infinite(Decimal('NaN'))+        False+        >>> ExtendedContext.is_infinite(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_infinite()++    def is_nan(self, a):+        """Return True if the operand is a qNaN or sNaN;+        otherwise return False.++        >>> ExtendedContext.is_nan(Decimal('2.50'))+        False+        >>> ExtendedContext.is_nan(Decimal('NaN'))+        True+        >>> ExtendedContext.is_nan(Decimal('-sNaN'))+        True+        >>> ExtendedContext.is_nan(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_nan()++    def is_normal(self, a):+        """Return True if the operand is a normal number;+        otherwise return False.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.is_normal(Decimal('2.50'))+        True+        >>> c.is_normal(Decimal('0.1E-999'))+        False+        >>> c.is_normal(Decimal('0.00'))+        False+        >>> c.is_normal(Decimal('-Inf'))+        False+        >>> c.is_normal(Decimal('NaN'))+        False+        >>> c.is_normal(1)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_normal(context=self)++    def is_qnan(self, a):+        """Return True if the operand is a quiet NaN; otherwise return False.++        >>> ExtendedContext.is_qnan(Decimal('2.50'))+        False+        >>> ExtendedContext.is_qnan(Decimal('NaN'))+        True+        >>> ExtendedContext.is_qnan(Decimal('sNaN'))+        False+        >>> ExtendedContext.is_qnan(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_qnan()++    def is_signed(self, a):+        """Return True if the operand is negative; otherwise return False.++        >>> ExtendedContext.is_signed(Decimal('2.50'))+        False+        >>> ExtendedContext.is_signed(Decimal('-12'))+        True+        >>> ExtendedContext.is_signed(Decimal('-0'))+        True+        >>> ExtendedContext.is_signed(8)+        False+        >>> ExtendedContext.is_signed(-8)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_signed()++    def is_snan(self, a):+        """Return True if the operand is a signaling NaN;+        otherwise return False.++        >>> ExtendedContext.is_snan(Decimal('2.50'))+        False+        >>> ExtendedContext.is_snan(Decimal('NaN'))+        False+        >>> ExtendedContext.is_snan(Decimal('sNaN'))+        True+        >>> ExtendedContext.is_snan(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_snan()++    def is_subnormal(self, a):+        """Return True if the operand is subnormal; otherwise return False.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.is_subnormal(Decimal('2.50'))+        False+        >>> c.is_subnormal(Decimal('0.1E-999'))+        True+        >>> c.is_subnormal(Decimal('0.00'))+        False+        >>> c.is_subnormal(Decimal('-Inf'))+        False+        >>> c.is_subnormal(Decimal('NaN'))+        False+        >>> c.is_subnormal(1)+        False+        """+        a = _convert_other(a, raiseit=True)+        return a.is_subnormal(context=self)++    def is_zero(self, a):+        """Return True if the operand is a zero; otherwise return False.++        >>> ExtendedContext.is_zero(Decimal('0'))+        True+        >>> ExtendedContext.is_zero(Decimal('2.50'))+        False+        >>> ExtendedContext.is_zero(Decimal('-0E+2'))+        True+        >>> ExtendedContext.is_zero(1)+        False+        >>> ExtendedContext.is_zero(0)+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.is_zero()++    def ln(self, a):+        """Returns the natural (base e) logarithm of the operand.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.ln(Decimal('0'))+        Decimal('-Infinity')+        >>> c.ln(Decimal('1.000'))+        Decimal('0')+        >>> c.ln(Decimal('2.71828183'))+        Decimal('1.00000000')+        >>> c.ln(Decimal('10'))+        Decimal('2.30258509')+        >>> c.ln(Decimal('+Infinity'))+        Decimal('Infinity')+        >>> c.ln(1)+        Decimal('0')+        """+        a = _convert_other(a, raiseit=True)+        return a.ln(context=self)++    def log10(self, a):+        """Returns the base 10 logarithm of the operand.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.log10(Decimal('0'))+        Decimal('-Infinity')+        >>> c.log10(Decimal('0.001'))+        Decimal('-3')+        >>> c.log10(Decimal('1.000'))+        Decimal('0')+        >>> c.log10(Decimal('2'))+        Decimal('0.301029996')+        >>> c.log10(Decimal('10'))+        Decimal('1')+        >>> c.log10(Decimal('70'))+        Decimal('1.84509804')+        >>> c.log10(Decimal('+Infinity'))+        Decimal('Infinity')+        >>> c.log10(0)+        Decimal('-Infinity')+        >>> c.log10(1)+        Decimal('0')+        """+        a = _convert_other(a, raiseit=True)+        return a.log10(context=self)++    def logb(self, a):+        """ Returns the exponent of the magnitude of the operand's MSD.++        The result is the integer which is the exponent of the magnitude+        of the most significant digit of the operand (as though the+        operand were truncated to a single digit while maintaining the+        value of that digit and without limiting the resulting exponent).++        >>> ExtendedContext.logb(Decimal('250'))+        Decimal('2')+        >>> ExtendedContext.logb(Decimal('2.50'))+        Decimal('0')+        >>> ExtendedContext.logb(Decimal('0.03'))+        Decimal('-2')+        >>> ExtendedContext.logb(Decimal('0'))+        Decimal('-Infinity')+        >>> ExtendedContext.logb(1)+        Decimal('0')+        >>> ExtendedContext.logb(10)+        Decimal('1')+        >>> ExtendedContext.logb(100)+        Decimal('2')+        """+        a = _convert_other(a, raiseit=True)+        return a.logb(context=self)++    def logical_and(self, a, b):+        """Applies the logical operation 'and' between each operand's digits.++        The operands must be both logical numbers.++        >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1'))+        Decimal('0')+        >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010'))+        Decimal('1000')+        >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10'))+        Decimal('10')+        >>> ExtendedContext.logical_and(110, 1101)+        Decimal('100')+        >>> ExtendedContext.logical_and(Decimal(110), 1101)+        Decimal('100')+        >>> ExtendedContext.logical_and(110, Decimal(1101))+        Decimal('100')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_and(b, context=self)++    def logical_invert(self, a):+        """Invert all the digits in the operand.++        The operand must be a logical number.++        >>> ExtendedContext.logical_invert(Decimal('0'))+        Decimal('111111111')+        >>> ExtendedContext.logical_invert(Decimal('1'))+        Decimal('111111110')+        >>> ExtendedContext.logical_invert(Decimal('111111111'))+        Decimal('0')+        >>> ExtendedContext.logical_invert(Decimal('101010101'))+        Decimal('10101010')+        >>> ExtendedContext.logical_invert(1101)+        Decimal('111110010')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_invert(context=self)++    def logical_or(self, a, b):+        """Applies the logical operation 'or' between each operand's digits.++        The operands must be both logical numbers.++        >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0'))+        Decimal('1')+        >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010'))+        Decimal('1110')+        >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10'))+        Decimal('1110')+        >>> ExtendedContext.logical_or(110, 1101)+        Decimal('1111')+        >>> ExtendedContext.logical_or(Decimal(110), 1101)+        Decimal('1111')+        >>> ExtendedContext.logical_or(110, Decimal(1101))+        Decimal('1111')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_or(b, context=self)++    def logical_xor(self, a, b):+        """Applies the logical operation 'xor' between each operand's digits.++        The operands must be both logical numbers.++        >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0'))+        Decimal('1')+        >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1'))+        Decimal('0')+        >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010'))+        Decimal('110')+        >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10'))+        Decimal('1101')+        >>> ExtendedContext.logical_xor(110, 1101)+        Decimal('1011')+        >>> ExtendedContext.logical_xor(Decimal(110), 1101)+        Decimal('1011')+        >>> ExtendedContext.logical_xor(110, Decimal(1101))+        Decimal('1011')+        """+        a = _convert_other(a, raiseit=True)+        return a.logical_xor(b, context=self)++    def max(self, a, b):+        """max compares two values numerically and returns the maximum.++        If either operand is a NaN then the general rules apply.+        Otherwise, the operands are compared as though by the compare+        operation.  If they are numerically equal then the left-hand operand+        is chosen as the result.  Otherwise the maximum (closer to positive+        infinity) of the two operands is chosen as the result.++        >>> ExtendedContext.max(Decimal('3'), Decimal('2'))+        Decimal('3')+        >>> ExtendedContext.max(Decimal('-10'), Decimal('3'))+        Decimal('3')+        >>> ExtendedContext.max(Decimal('1.0'), Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.max(Decimal('7'), Decimal('NaN'))+        Decimal('7')+        >>> ExtendedContext.max(1, 2)+        Decimal('2')+        >>> ExtendedContext.max(Decimal(1), 2)+        Decimal('2')+        >>> ExtendedContext.max(1, Decimal(2))+        Decimal('2')+        """+        a = _convert_other(a, raiseit=True)+        return a.max(b, context=self)++    def max_mag(self, a, b):+        """Compares the values numerically with their sign ignored.++        >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN'))+        Decimal('7')+        >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10'))+        Decimal('-10')+        >>> ExtendedContext.max_mag(1, -2)+        Decimal('-2')+        >>> ExtendedContext.max_mag(Decimal(1), -2)+        Decimal('-2')+        >>> ExtendedContext.max_mag(1, Decimal(-2))+        Decimal('-2')+        """+        a = _convert_other(a, raiseit=True)+        return a.max_mag(b, context=self)++    def min(self, a, b):+        """min compares two values numerically and returns the minimum.++        If either operand is a NaN then the general rules apply.+        Otherwise, the operands are compared as though by the compare+        operation.  If they are numerically equal then the left-hand operand+        is chosen as the result.  Otherwise the minimum (closer to negative+        infinity) of the two operands is chosen as the result.++        >>> ExtendedContext.min(Decimal('3'), Decimal('2'))+        Decimal('2')+        >>> ExtendedContext.min(Decimal('-10'), Decimal('3'))+        Decimal('-10')+        >>> ExtendedContext.min(Decimal('1.0'), Decimal('1'))+        Decimal('1.0')+        >>> ExtendedContext.min(Decimal('7'), Decimal('NaN'))+        Decimal('7')+        >>> ExtendedContext.min(1, 2)+        Decimal('1')+        >>> ExtendedContext.min(Decimal(1), 2)+        Decimal('1')+        >>> ExtendedContext.min(1, Decimal(29))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.min(b, context=self)++    def min_mag(self, a, b):+        """Compares the values numerically with their sign ignored.++        >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2'))+        Decimal('-2')+        >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN'))+        Decimal('-3')+        >>> ExtendedContext.min_mag(1, -2)+        Decimal('1')+        >>> ExtendedContext.min_mag(Decimal(1), -2)+        Decimal('1')+        >>> ExtendedContext.min_mag(1, Decimal(-2))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.min_mag(b, context=self)++    def minus(self, a):+        """Minus corresponds to unary prefix minus in Python.++        The operation is evaluated using the same rules as subtract; the+        operation minus(a) is calculated as subtract('0', a) where the '0'+        has the same exponent as the operand.++        >>> ExtendedContext.minus(Decimal('1.3'))+        Decimal('-1.3')+        >>> ExtendedContext.minus(Decimal('-1.3'))+        Decimal('1.3')+        >>> ExtendedContext.minus(1)+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.__neg__(context=self)++    def multiply(self, a, b):+        """multiply multiplies two operands.++        If either operand is a special value then the general rules apply.+        Otherwise, the operands are multiplied together+        ('long multiplication'), resulting in a number which may be as long as+        the sum of the lengths of the two operands.++        >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3'))+        Decimal('3.60')+        >>> ExtendedContext.multiply(Decimal('7'), Decimal('3'))+        Decimal('21')+        >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8'))+        Decimal('0.72')+        >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0'))+        Decimal('-0.0')+        >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321'))+        Decimal('4.28135971E+11')+        >>> ExtendedContext.multiply(7, 7)+        Decimal('49')+        >>> ExtendedContext.multiply(Decimal(7), 7)+        Decimal('49')+        >>> ExtendedContext.multiply(7, Decimal(7))+        Decimal('49')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__mul__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def next_minus(self, a):+        """Returns the largest representable number smaller than a.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> ExtendedContext.next_minus(Decimal('1'))+        Decimal('0.999999999')+        >>> c.next_minus(Decimal('1E-1007'))+        Decimal('0E-1007')+        >>> ExtendedContext.next_minus(Decimal('-1.00000003'))+        Decimal('-1.00000004')+        >>> c.next_minus(Decimal('Infinity'))+        Decimal('9.99999999E+999')+        >>> c.next_minus(1)+        Decimal('0.999999999')+        """+        a = _convert_other(a, raiseit=True)+        return a.next_minus(context=self)++    def next_plus(self, a):+        """Returns the smallest representable number larger than a.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> ExtendedContext.next_plus(Decimal('1'))+        Decimal('1.00000001')+        >>> c.next_plus(Decimal('-1E-1007'))+        Decimal('-0E-1007')+        >>> ExtendedContext.next_plus(Decimal('-1.00000003'))+        Decimal('-1.00000002')+        >>> c.next_plus(Decimal('-Infinity'))+        Decimal('-9.99999999E+999')+        >>> c.next_plus(1)+        Decimal('1.00000001')+        """+        a = _convert_other(a, raiseit=True)+        return a.next_plus(context=self)++    def next_toward(self, a, b):+        """Returns the number closest to a, in direction towards b.++        The result is the closest representable number from the first+        operand (but not the first operand) that is in the direction+        towards the second operand, unless the operands have the same+        value.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.next_toward(Decimal('1'), Decimal('2'))+        Decimal('1.00000001')+        >>> c.next_toward(Decimal('-1E-1007'), Decimal('1'))+        Decimal('-0E-1007')+        >>> c.next_toward(Decimal('-1.00000003'), Decimal('0'))+        Decimal('-1.00000002')+        >>> c.next_toward(Decimal('1'), Decimal('0'))+        Decimal('0.999999999')+        >>> c.next_toward(Decimal('1E-1007'), Decimal('-100'))+        Decimal('0E-1007')+        >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10'))+        Decimal('-1.00000004')+        >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000'))+        Decimal('-0.00')+        >>> c.next_toward(0, 1)+        Decimal('1E-1007')+        >>> c.next_toward(Decimal(0), 1)+        Decimal('1E-1007')+        >>> c.next_toward(0, Decimal(1))+        Decimal('1E-1007')+        """+        a = _convert_other(a, raiseit=True)+        return a.next_toward(b, context=self)++    def normalize(self, a):+        """normalize reduces an operand to its simplest form.++        Essentially a plus operation with all trailing zeros removed from the+        result.++        >>> ExtendedContext.normalize(Decimal('2.1'))+        Decimal('2.1')+        >>> ExtendedContext.normalize(Decimal('-2.0'))+        Decimal('-2')+        >>> ExtendedContext.normalize(Decimal('1.200'))+        Decimal('1.2')+        >>> ExtendedContext.normalize(Decimal('-120'))+        Decimal('-1.2E+2')+        >>> ExtendedContext.normalize(Decimal('120.00'))+        Decimal('1.2E+2')+        >>> ExtendedContext.normalize(Decimal('0.00'))+        Decimal('0')+        >>> ExtendedContext.normalize(6)+        Decimal('6')+        """+        a = _convert_other(a, raiseit=True)+        return a.normalize(context=self)++    def number_class(self, a):+        """Returns an indication of the class of the operand.++        The class is one of the following strings:+          -sNaN+          -NaN+          -Infinity+          -Normal+          -Subnormal+          -Zero+          +Zero+          +Subnormal+          +Normal+          +Infinity++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.number_class(Decimal('Infinity'))+        '+Infinity'+        >>> c.number_class(Decimal('1E-10'))+        '+Normal'+        >>> c.number_class(Decimal('2.50'))+        '+Normal'+        >>> c.number_class(Decimal('0.1E-999'))+        '+Subnormal'+        >>> c.number_class(Decimal('0'))+        '+Zero'+        >>> c.number_class(Decimal('-0'))+        '-Zero'+        >>> c.number_class(Decimal('-0.1E-999'))+        '-Subnormal'+        >>> c.number_class(Decimal('-1E-10'))+        '-Normal'+        >>> c.number_class(Decimal('-2.50'))+        '-Normal'+        >>> c.number_class(Decimal('-Infinity'))+        '-Infinity'+        >>> c.number_class(Decimal('NaN'))+        'NaN'+        >>> c.number_class(Decimal('-NaN'))+        'NaN'+        >>> c.number_class(Decimal('sNaN'))+        'sNaN'+        >>> c.number_class(123)+        '+Normal'+        """+        a = _convert_other(a, raiseit=True)+        return a.number_class(context=self)++    def plus(self, a):+        """Plus corresponds to unary prefix plus in Python.++        The operation is evaluated using the same rules as add; the+        operation plus(a) is calculated as add('0', a) where the '0'+        has the same exponent as the operand.++        >>> ExtendedContext.plus(Decimal('1.3'))+        Decimal('1.3')+        >>> ExtendedContext.plus(Decimal('-1.3'))+        Decimal('-1.3')+        >>> ExtendedContext.plus(-1)+        Decimal('-1')+        """+        a = _convert_other(a, raiseit=True)+        return a.__pos__(context=self)++    def power(self, a, b, modulo=None):+        """Raises a to the power of b, to modulo if given.++        With two arguments, compute a**b.  If a is negative then b+        must be integral.  The result will be inexact unless b is+        integral and the result is finite and can be expressed exactly+        in 'precision' digits.++        With three arguments, compute (a**b) % modulo.  For the+        three argument form, the following restrictions on the+        arguments hold:++         - all three arguments must be integral+         - b must be nonnegative+         - at least one of a or b must be nonzero+         - modulo must be nonzero and have at most 'precision' digits++        The result of pow(a, b, modulo) is identical to the result+        that would be obtained by computing (a**b) % modulo with+        unbounded precision, but is computed more efficiently.  It is+        always exact.++        >>> c = ExtendedContext.copy()+        >>> c.Emin = -999+        >>> c.Emax = 999+        >>> c.power(Decimal('2'), Decimal('3'))+        Decimal('8')+        >>> c.power(Decimal('-2'), Decimal('3'))+        Decimal('-8')+        >>> c.power(Decimal('2'), Decimal('-3'))+        Decimal('0.125')+        >>> c.power(Decimal('1.7'), Decimal('8'))+        Decimal('69.7575744')+        >>> c.power(Decimal('10'), Decimal('0.301029996'))+        Decimal('2.00000000')+        >>> c.power(Decimal('Infinity'), Decimal('-1'))+        Decimal('0')+        >>> c.power(Decimal('Infinity'), Decimal('0'))+        Decimal('1')+        >>> c.power(Decimal('Infinity'), Decimal('1'))+        Decimal('Infinity')+        >>> c.power(Decimal('-Infinity'), Decimal('-1'))+        Decimal('-0')+        >>> c.power(Decimal('-Infinity'), Decimal('0'))+        Decimal('1')+        >>> c.power(Decimal('-Infinity'), Decimal('1'))+        Decimal('-Infinity')+        >>> c.power(Decimal('-Infinity'), Decimal('2'))+        Decimal('Infinity')+        >>> c.power(Decimal('0'), Decimal('0'))+        Decimal('NaN')++        >>> c.power(Decimal('3'), Decimal('7'), Decimal('16'))+        Decimal('11')+        >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16'))+        Decimal('-11')+        >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16'))+        Decimal('1')+        >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16'))+        Decimal('11')+        >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789'))+        Decimal('11729830')+        >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729'))+        Decimal('-0')+        >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537'))+        Decimal('1')+        >>> ExtendedContext.power(7, 7)+        Decimal('823543')+        >>> ExtendedContext.power(Decimal(7), 7)+        Decimal('823543')+        >>> ExtendedContext.power(7, Decimal(7), 2)+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__pow__(b, modulo, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def quantize(self, a, b):+        """Returns a value equal to 'a' (rounded), having the exponent of 'b'.++        The coefficient of the result is derived from that of the left-hand+        operand.  It may be rounded using the current rounding setting (if the+        exponent is being increased), multiplied by a positive power of ten (if+        the exponent is being decreased), or is unchanged (if the exponent is+        already equal to that of the right-hand operand).++        Unlike other operations, if the length of the coefficient after the+        quantize operation would be greater than precision then an Invalid+        operation condition is raised.  This guarantees that, unless there is+        an error condition, the exponent of the result of a quantize is always+        equal to that of the right-hand operand.++        Also unlike other operations, quantize will never raise Underflow, even+        if the result is subnormal and inexact.++        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001'))+        Decimal('2.170')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01'))+        Decimal('2.17')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1'))+        Decimal('2.2')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0'))+        Decimal('2')+        >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1'))+        Decimal('0E+1')+        >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity'))+        Decimal('-Infinity')+        >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity'))+        Decimal('NaN')+        >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1'))+        Decimal('-0')+        >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5'))+        Decimal('-0E+5')+        >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2'))+        Decimal('NaN')+        >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2'))+        Decimal('NaN')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1'))+        Decimal('217.0')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0'))+        Decimal('217')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1'))+        Decimal('2.2E+2')+        >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2'))+        Decimal('2E+2')+        >>> ExtendedContext.quantize(1, 2)+        Decimal('1')+        >>> ExtendedContext.quantize(Decimal(1), 2)+        Decimal('1')+        >>> ExtendedContext.quantize(1, Decimal(2))+        Decimal('1')+        """+        a = _convert_other(a, raiseit=True)+        return a.quantize(b, context=self)++    def radix(self):+        """Just returns 10, as this is Decimal, :)++        >>> ExtendedContext.radix()+        Decimal('10')+        """+        return Decimal(10)++    def remainder(self, a, b):+        """Returns the remainder from integer division.++        The result is the residue of the dividend after the operation of+        calculating integer division as described for divide-integer, rounded+        to precision digits if necessary.  The sign of the result, if+        non-zero, is the same as that of the original dividend.++        This operation will fail under the same conditions as integer division+        (that is, if integer division on the same two operands would fail, the+        remainder cannot be calculated).++        >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3'))+        Decimal('2.1')+        >>> ExtendedContext.remainder(Decimal('10'), Decimal('3'))+        Decimal('1')+        >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3'))+        Decimal('-1')+        >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1'))+        Decimal('0.2')+        >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3'))+        Decimal('0.1')+        >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3'))+        Decimal('1.0')+        >>> ExtendedContext.remainder(22, 6)+        Decimal('4')+        >>> ExtendedContext.remainder(Decimal(22), 6)+        Decimal('4')+        >>> ExtendedContext.remainder(22, Decimal(6))+        Decimal('4')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__mod__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def remainder_near(self, a, b):+        """Returns to be "a - b * n", where n is the integer nearest the exact+        value of "x / b" (if two integers are equally near then the even one+        is chosen).  If the result is equal to 0 then its sign will be the+        sign of a.++        This operation will fail under the same conditions as integer division+        (that is, if integer division on the same two operands would fail, the+        remainder cannot be calculated).++        >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3'))+        Decimal('-0.9')+        >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6'))+        Decimal('-2')+        >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3'))+        Decimal('1')+        >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3'))+        Decimal('-1')+        >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1'))+        Decimal('0.2')+        >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3'))+        Decimal('0.1')+        >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3'))+        Decimal('-0.3')+        >>> ExtendedContext.remainder_near(3, 11)+        Decimal('3')+        >>> ExtendedContext.remainder_near(Decimal(3), 11)+        Decimal('3')+        >>> ExtendedContext.remainder_near(3, Decimal(11))+        Decimal('3')+        """+        a = _convert_other(a, raiseit=True)+        return a.remainder_near(b, context=self)++    def rotate(self, a, b):+        """Returns a rotated copy of a, b times.++        The coefficient of the result is a rotated copy of the digits in+        the coefficient of the first operand.  The number of places of+        rotation is taken from the absolute value of the second operand,+        with the rotation being to the left if the second operand is+        positive or to the right otherwise.++        >>> ExtendedContext.rotate(Decimal('34'), Decimal('8'))+        Decimal('400000003')+        >>> ExtendedContext.rotate(Decimal('12'), Decimal('9'))+        Decimal('12')+        >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2'))+        Decimal('891234567')+        >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0'))+        Decimal('123456789')+        >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2'))+        Decimal('345678912')+        >>> ExtendedContext.rotate(1333333, 1)+        Decimal('13333330')+        >>> ExtendedContext.rotate(Decimal(1333333), 1)+        Decimal('13333330')+        >>> ExtendedContext.rotate(1333333, Decimal(1))+        Decimal('13333330')+        """+        a = _convert_other(a, raiseit=True)+        return a.rotate(b, context=self)++    def same_quantum(self, a, b):+        """Returns True if the two operands have the same exponent.++        The result is never affected by either the sign or the coefficient of+        either operand.++        >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001'))+        False+        >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01'))+        True+        >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1'))+        False+        >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf'))+        True+        >>> ExtendedContext.same_quantum(10000, -1)+        True+        >>> ExtendedContext.same_quantum(Decimal(10000), -1)+        True+        >>> ExtendedContext.same_quantum(10000, Decimal(-1))+        True+        """+        a = _convert_other(a, raiseit=True)+        return a.same_quantum(b)++    def scaleb (self, a, b):+        """Returns the first operand after adding the second value its exp.++        >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2'))+        Decimal('0.0750')+        >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0'))+        Decimal('7.50')+        >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3'))+        Decimal('7.50E+3')+        >>> ExtendedContext.scaleb(1, 4)+        Decimal('1E+4')+        >>> ExtendedContext.scaleb(Decimal(1), 4)+        Decimal('1E+4')+        >>> ExtendedContext.scaleb(1, Decimal(4))+        Decimal('1E+4')+        """+        a = _convert_other(a, raiseit=True)+        return a.scaleb(b, context=self)++    def shift(self, a, b):+        """Returns a shifted copy of a, b times.++        The coefficient of the result is a shifted copy of the digits+        in the coefficient of the first operand.  The number of places+        to shift is taken from the absolute value of the second operand,+        with the shift being to the left if the second operand is+        positive or to the right otherwise.  Digits shifted into the+        coefficient are zeros.++        >>> ExtendedContext.shift(Decimal('34'), Decimal('8'))+        Decimal('400000000')+        >>> ExtendedContext.shift(Decimal('12'), Decimal('9'))+        Decimal('0')+        >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2'))+        Decimal('1234567')+        >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0'))+        Decimal('123456789')+        >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2'))+        Decimal('345678900')+        >>> ExtendedContext.shift(88888888, 2)+        Decimal('888888800')+        >>> ExtendedContext.shift(Decimal(88888888), 2)+        Decimal('888888800')+        >>> ExtendedContext.shift(88888888, Decimal(2))+        Decimal('888888800')+        """+        a = _convert_other(a, raiseit=True)+        return a.shift(b, context=self)++    def sqrt(self, a):+        """Square root of a non-negative number to context precision.++        If the result must be inexact, it is rounded using the round-half-even+        algorithm.++        >>> ExtendedContext.sqrt(Decimal('0'))+        Decimal('0')+        >>> ExtendedContext.sqrt(Decimal('-0'))+        Decimal('-0')+        >>> ExtendedContext.sqrt(Decimal('0.39'))+        Decimal('0.624499800')+        >>> ExtendedContext.sqrt(Decimal('100'))+        Decimal('10')+        >>> ExtendedContext.sqrt(Decimal('1'))+        Decimal('1')+        >>> ExtendedContext.sqrt(Decimal('1.0'))+        Decimal('1.0')+        >>> ExtendedContext.sqrt(Decimal('1.00'))+        Decimal('1.0')+        >>> ExtendedContext.sqrt(Decimal('7'))+        Decimal('2.64575131')+        >>> ExtendedContext.sqrt(Decimal('10'))+        Decimal('3.16227766')+        >>> ExtendedContext.sqrt(2)+        Decimal('1.41421356')+        >>> ExtendedContext.prec+        9+        """+        a = _convert_other(a, raiseit=True)+        return a.sqrt(context=self)++    def subtract(self, a, b):+        """Return the difference between the two operands.++        >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07'))+        Decimal('0.23')+        >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30'))+        Decimal('0.00')+        >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07'))+        Decimal('-0.77')+        >>> ExtendedContext.subtract(8, 5)+        Decimal('3')+        >>> ExtendedContext.subtract(Decimal(8), 5)+        Decimal('3')+        >>> ExtendedContext.subtract(8, Decimal(5))+        Decimal('3')+        """+        a = _convert_other(a, raiseit=True)+        r = a.__sub__(b, context=self)+        if r is NotImplemented:+            raise TypeError("Unable to convert %s to Decimal" % b)+        else:+            return r++    def to_eng_string(self, a):+        """Convert to a string, using engineering notation if an exponent is needed.++        Engineering notation has an exponent which is a multiple of 3.  This+        can leave up to 3 digits to the left of the decimal place and may+        require the addition of either one or two trailing zeros.++        The operation is not affected by the context.++        >>> ExtendedContext.to_eng_string(Decimal('123E+1'))+        '1.23E+3'+        >>> ExtendedContext.to_eng_string(Decimal('123E+3'))+        '123E+3'+        >>> ExtendedContext.to_eng_string(Decimal('123E-10'))+        '12.3E-9'+        >>> ExtendedContext.to_eng_string(Decimal('-123E-12'))+        '-123E-12'+        >>> ExtendedContext.to_eng_string(Decimal('7E-7'))+        '700E-9'+        >>> ExtendedContext.to_eng_string(Decimal('7E+1'))+        '70'+        >>> ExtendedContext.to_eng_string(Decimal('0E+1'))+        '0.00E+3'++        """+        a = _convert_other(a, raiseit=True)+        return a.to_eng_string(context=self)++    def to_sci_string(self, a):+        """Converts a number to a string, using scientific notation.++        The operation is not affected by the context.+        """+        a = _convert_other(a, raiseit=True)+        return a.__str__(context=self)++    def to_integral_exact(self, a):+        """Rounds to an integer.++        When the operand has a negative exponent, the result is the same+        as using the quantize() operation using the given operand as the+        left-hand-operand, 1E+0 as the right-hand-operand, and the precision+        of the operand as the precision setting; Inexact and Rounded flags+        are allowed in this operation.  The rounding mode is taken from the+        context.++        >>> ExtendedContext.to_integral_exact(Decimal('2.1'))+        Decimal('2')+        >>> ExtendedContext.to_integral_exact(Decimal('100'))+        Decimal('100')+        >>> ExtendedContext.to_integral_exact(Decimal('100.0'))+        Decimal('100')+        >>> ExtendedContext.to_integral_exact(Decimal('101.5'))+        Decimal('102')+        >>> ExtendedContext.to_integral_exact(Decimal('-101.5'))+        Decimal('-102')+        >>> ExtendedContext.to_integral_exact(Decimal('10E+5'))+        Decimal('1.0E+6')+        >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77'))+        Decimal('7.89E+77')+        >>> ExtendedContext.to_integral_exact(Decimal('-Inf'))+        Decimal('-Infinity')+        """+        a = _convert_other(a, raiseit=True)+        return a.to_integral_exact(context=self)++    def to_integral_value(self, a):+        """Rounds to an integer.++        When the operand has a negative exponent, the result is the same+        as using the quantize() operation using the given operand as the+        left-hand-operand, 1E+0 as the right-hand-operand, and the precision+        of the operand as the precision setting, except that no flags will+        be set.  The rounding mode is taken from the context.++        >>> ExtendedContext.to_integral_value(Decimal('2.1'))+        Decimal('2')+        >>> ExtendedContext.to_integral_value(Decimal('100'))+        Decimal('100')+        >>> ExtendedContext.to_integral_value(Decimal('100.0'))+        Decimal('100')+        >>> ExtendedContext.to_integral_value(Decimal('101.5'))+        Decimal('102')+        >>> ExtendedContext.to_integral_value(Decimal('-101.5'))+        Decimal('-102')+        >>> ExtendedContext.to_integral_value(Decimal('10E+5'))+        Decimal('1.0E+6')+        >>> ExtendedContext.to_integral_value(Decimal('7.89E+77'))+        Decimal('7.89E+77')+        >>> ExtendedContext.to_integral_value(Decimal('-Inf'))+        Decimal('-Infinity')+        """+        a = _convert_other(a, raiseit=True)+        return a.to_integral_value(context=self)++    # the method name changed, but we provide also the old one, for compatibility+    to_integral = to_integral_value++class _WorkRep(object):+    __slots__ = ('sign','int','exp')+    # sign: 0 or 1+    # int:  int+    # exp:  None, int, or string++    def __init__(self, value=None):+        if value is None:+            self.sign = None+            self.int = 0+            self.exp = None+        elif isinstance(value, Decimal):+            self.sign = value._sign+            self.int = int(value._int)+            self.exp = value._exp+        else:+            # assert isinstance(value, tuple)+            self.sign = value[0]+            self.int = value[1]+            self.exp = value[2]++    def __repr__(self):+        return "(%r, %r, %r)" % (self.sign, self.int, self.exp)++    __str__ = __repr__++++def _normalize(op1, op2, prec = 0):+    """Normalizes op1, op2 to have the same exp and length of coefficient.++    Done during addition.+    """+    if op1.exp < op2.exp:+        tmp = op2+        other = op1+    else:+        tmp = op1+        other = op2++    # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1).+    # Then adding 10**exp to tmp has the same effect (after rounding)+    # as adding any positive quantity smaller than 10**exp; similarly+    # for subtraction.  So if other is smaller than 10**exp we replace+    # it with 10**exp.  This avoids tmp.exp - other.exp getting too large.+    tmp_len = len(str(tmp.int))+    other_len = len(str(other.int))+    exp = tmp.exp + min(-1, tmp_len - prec - 2)+    if other_len + other.exp - 1 < exp:+        other.int = 1+        other.exp = exp++    tmp.int *= 10 ** (tmp.exp - other.exp)+    tmp.exp = other.exp+    return op1, op2++##### Integer arithmetic functions used by ln, log10, exp and __pow__ #####++_nbits = int.bit_length++def _decimal_lshift_exact(n, e):+    """ Given integers n and e, return n * 10**e if it's an integer, else None.++    The computation is designed to avoid computing large powers of 10+    unnecessarily.++    >>> _decimal_lshift_exact(3, 4)+    30000+    >>> _decimal_lshift_exact(300, -999999999)  # returns None++    """+    if n == 0:+        return 0+    elif e >= 0:+        return n * 10**e+    else:+        # val_n = largest power of 10 dividing n.+        str_n = str(abs(n))+        val_n = len(str_n) - len(str_n.rstrip('0'))+        return None if val_n < -e else n // 10**-e++def _sqrt_nearest(n, a):+    """Closest integer to the square root of the positive integer n.  a is+    an initial approximation to the square root.  Any positive integer+    will do for a, but the closer a is to the square root of n the+    faster convergence will be.++    """+    if n <= 0 or a <= 0:+        raise ValueError("Both arguments to _sqrt_nearest should be positive.")++    b=0+    while a != b:+        b, a = a, a--n//a>>1+    return a++def _rshift_nearest(x, shift):+    """Given an integer x and a nonnegative integer shift, return closest+    integer to x / 2**shift; use round-to-even in case of a tie.++    """+    b, q = 1 << shift, x >> shift+    return q + (2*(x & (b-1)) + (q&1) > b)++def _div_nearest(a, b):+    """Closest integer to a/b, a and b positive integers; rounds to even+    in the case of a tie.++    """+    q, r = divmod(a, b)+    return q + (2*r + (q&1) > b)++def _ilog(x, M, L = 8):+    """Integer approximation to M*log(x/M), with absolute error boundable+    in terms only of x/M.++    Given positive integers x and M, return an integer approximation to+    M * log(x/M).  For L = 8 and 0.1 <= x/M <= 10 the difference+    between the approximation and the exact result is at most 22.  For+    L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15.  In+    both cases these are upper bounds on the error; it will usually be+    much smaller."""++    # The basic algorithm is the following: let log1p be the function+    # log1p(x) = log(1+x).  Then log(x/M) = log1p((x-M)/M).  We use+    # the reduction+    #+    #    log1p(y) = 2*log1p(y/(1+sqrt(1+y)))+    #+    # repeatedly until the argument to log1p is small (< 2**-L in+    # absolute value).  For small y we can use the Taylor series+    # expansion+    #+    #    log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T+    #+    # truncating at T such that y**T is small enough.  The whole+    # computation is carried out in a form of fixed-point arithmetic,+    # with a real number z being represented by an integer+    # approximation to z*M.  To avoid loss of precision, the y below+    # is actually an integer approximation to 2**R*y*M, where R is the+    # number of reductions performed so far.++    y = x-M+    # argument reduction; R = number of reductions performed+    R = 0+    while (R <= L and abs(y) << L-R >= M or+           R > L and abs(y) >> R-L >= M):+        y = _div_nearest((M*y) << 1,+                         M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M))+        R += 1++    # Taylor series with T terms+    T = -int(-10*len(str(M))//(3*L))+    yshift = _rshift_nearest(y, R)+    w = _div_nearest(M, T)+    for k in range(T-1, 0, -1):+        w = _div_nearest(M, k) - _div_nearest(yshift*w, M)++    return _div_nearest(w*y, M)++def _dlog10(c, e, p):+    """Given integers c, e and p with c > 0, p >= 0, compute an integer+    approximation to 10**p * log10(c*10**e), with an absolute error of+    at most 1.  Assumes that c*10**e is not exactly 1."""++    # increase precision by 2; compensate for this by dividing+    # final result by 100+    p += 2++    # write c*10**e as d*10**f with either:+    #   f >= 0 and 1 <= d <= 10, or+    #   f <= 0 and 0.1 <= d <= 1.+    # Thus for c*10**e close to 1, f = 0+    l = len(str(c))+    f = e+l - (e+l >= 1)++    if p > 0:+        M = 10**p+        k = e+p-f+        if k >= 0:+            c *= 10**k+        else:+            c = _div_nearest(c, 10**-k)++        log_d = _ilog(c, M) # error < 5 + 22 = 27+        log_10 = _log10_digits(p) # error < 1+        log_d = _div_nearest(log_d*M, log_10)+        log_tenpower = f*M # exact+    else:+        log_d = 0  # error < 2.31+        log_tenpower = _div_nearest(f, 10**-p) # error < 0.5++    return _div_nearest(log_tenpower+log_d, 100)++def _dlog(c, e, p):+    """Given integers c, e and p with c > 0, compute an integer+    approximation to 10**p * log(c*10**e), with an absolute error of+    at most 1.  Assumes that c*10**e is not exactly 1."""++    # Increase precision by 2. The precision increase is compensated+    # for at the end with a division by 100.+    p += 2++    # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10,+    # or f <= 0 and 0.1 <= d <= 1.  Then we can compute 10**p * log(c*10**e)+    # as 10**p * log(d) + 10**p*f * log(10).+    l = len(str(c))+    f = e+l - (e+l >= 1)++    # compute approximation to 10**p*log(d), with error < 27+    if p > 0:+        k = e+p-f+        if k >= 0:+            c *= 10**k+        else:+            c = _div_nearest(c, 10**-k)  # error of <= 0.5 in c++        # _ilog magnifies existing error in c by a factor of at most 10+        log_d = _ilog(c, 10**p) # error < 5 + 22 = 27+    else:+        # p <= 0: just approximate the whole thing by 0; error < 2.31+        log_d = 0++    # compute approximation to f*10**p*log(10), with error < 11.+    if f:+        extra = len(str(abs(f)))-1+        if p + extra >= 0:+            # error in f * _log10_digits(p+extra) < |f| * 1 = |f|+            # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11+            f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra)+        else:+            f_log_ten = 0+    else:+        f_log_ten = 0++    # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1+    return _div_nearest(f_log_ten + log_d, 100)++class _Log10Memoize(object):+    """Class to compute, store, and allow retrieval of, digits of the+    constant log(10) = 2.302585....  This constant is needed by+    Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__."""+    def __init__(self):+        self.digits = "23025850929940456840179914546843642076011014886"++    def getdigits(self, p):+        """Given an integer p >= 0, return floor(10**p)*log(10).++        For example, self.getdigits(3) returns 2302.+        """+        # digits are stored as a string, for quick conversion to+        # integer in the case that we've already computed enough+        # digits; the stored digits should always be correct+        # (truncated, not rounded to nearest).+        if p < 0:+            raise ValueError("p should be nonnegative")++        if p >= len(self.digits):+            # compute p+3, p+6, p+9, ... digits; continue until at+            # least one of the extra digits is nonzero+            extra = 3+            while True:+                # compute p+extra digits, correct to within 1ulp+                M = 10**(p+extra+2)+                digits = str(_div_nearest(_ilog(10*M, M), 100))+                if digits[-extra:] != '0'*extra:+                    break+                extra += 3+            # keep all reliable digits so far; remove trailing zeros+            # and next nonzero digit+            self.digits = digits.rstrip('0')[:-1]+        return int(self.digits[:p+1])++_log10_digits = _Log10Memoize().getdigits++def _iexp(x, M, L=8):+    """Given integers x and M, M > 0, such that x/M is small in absolute+    value, compute an integer approximation to M*exp(x/M).  For 0 <=+    x/M <= 2.4, the absolute error in the result is bounded by 60 (and+    is usually much smaller)."""++    # Algorithm: to compute exp(z) for a real number z, first divide z+    # by a suitable power R of 2 so that |z/2**R| < 2**-L.  Then+    # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor+    # series+    #+    #     expm1(x) = x + x**2/2! + x**3/3! + ...+    #+    # Now use the identity+    #+    #     expm1(2x) = expm1(x)*(expm1(x)+2)+    #+    # R times to compute the sequence expm1(z/2**R),+    # expm1(z/2**(R-1)), ... , exp(z/2), exp(z).++    # Find R such that x/2**R/M <= 2**-L+    R = _nbits((x<<L)//M)++    # Taylor series.  (2**L)**T > M+    T = -int(-10*len(str(M))//(3*L))+    y = _div_nearest(x, T)+    Mshift = M<<R+    for i in range(T-1, 0, -1):+        y = _div_nearest(x*(Mshift + y), Mshift * i)++    # Expansion+    for k in range(R-1, -1, -1):+        Mshift = M<<(k+2)+        y = _div_nearest(y*(y+Mshift), Mshift)++    return M+y++def _dexp(c, e, p):+    """Compute an approximation to exp(c*10**e), with p decimal places of+    precision.++    Returns integers d, f such that:++      10**(p-1) <= d <= 10**p, and+      (d-1)*10**f < exp(c*10**e) < (d+1)*10**f++    In other words, d*10**f is an approximation to exp(c*10**e) with p+    digits of precision, and with an error in d of at most 1.  This is+    almost, but not quite, the same as the error being < 1ulp: when d+    = 10**(p-1) the error could be up to 10 ulp."""++    # we'll call iexp with M = 10**(p+2), giving p+3 digits of precision+    p += 2++    # compute log(10) with extra precision = adjusted exponent of c*10**e+    extra = max(0, e + len(str(c)) - 1)+    q = p + extra++    # compute quotient c*10**e/(log(10)) = c*10**(e+q)/(log(10)*10**q),+    # rounding down+    shift = e+q+    if shift >= 0:+        cshift = c*10**shift+    else:+        cshift = c//10**-shift+    quot, rem = divmod(cshift, _log10_digits(q))++    # reduce remainder back to original precision+    rem = _div_nearest(rem, 10**extra)++    # error in result of _iexp < 120;  error after division < 0.62+    return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3++def _dpower(xc, xe, yc, ye, p):+    """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and+    y = yc*10**ye, compute x**y.  Returns a pair of integers (c, e) such that:++      10**(p-1) <= c <= 10**p, and+      (c-1)*10**e < x**y < (c+1)*10**e++    in other words, c*10**e is an approximation to x**y with p digits+    of precision, and with an error in c of at most 1.  (This is+    almost, but not quite, the same as the error being < 1ulp: when c+    == 10**(p-1) we can only guarantee error < 10ulp.)++    We assume that: x is positive and not equal to 1, and y is nonzero.+    """++    # Find b such that 10**(b-1) <= |y| <= 10**b+    b = len(str(abs(yc))) + ye++    # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point+    lxc = _dlog(xc, xe, p+b+1)++    # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1)+    shift = ye-b+    if shift >= 0:+        pc = lxc*yc*10**shift+    else:+        pc = _div_nearest(lxc*yc, 10**-shift)++    if pc == 0:+        # we prefer a result that isn't exactly 1; this makes it+        # easier to compute a correctly rounded result in __pow__+        if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1:+            coeff, exp = 10**(p-1)+1, 1-p+        else:+            coeff, exp = 10**p-1, -p+    else:+        coeff, exp = _dexp(pc, -(p+1), p+1)+        coeff = _div_nearest(coeff, 10)+        exp += 1++    return coeff, exp++def _log10_lb(c, correction = {+        '1': 100, '2': 70, '3': 53, '4': 40, '5': 31,+        '6': 23, '7': 16, '8': 10, '9': 5}):+    """Compute a lower bound for 100*log10(c) for a positive integer c."""+    if c <= 0:+        raise ValueError("The argument to _log10_lb should be nonnegative.")+    str_c = str(c)+    return 100*len(str_c) - correction[str_c[0]]++##### Helper Functions ####################################################++def _convert_other(other, raiseit=False, allow_float=False):+    """Convert other to Decimal.++    Verifies that it's ok to use in an implicit construction.+    If allow_float is true, allow conversion from float;  this+    is used in the comparison methods (__eq__ and friends).++    """+    if isinstance(other, Decimal):+        return other+    if isinstance(other, int):+        return Decimal(other)+    if allow_float and isinstance(other, float):+        return Decimal.from_float(other)++    if raiseit:+        raise TypeError("Unable to convert %s to Decimal" % other)+    return NotImplemented++def _convert_for_comparison(self, other, equality_op=False):+    """Given a Decimal instance self and a Python object other, return+    a pair (s, o) of Decimal instances such that "s op o" is+    equivalent to "self op other" for any of the 6 comparison+    operators "op".++    """+    if isinstance(other, Decimal):+        return self, other++    # Comparison with a Rational instance (also includes integers):+    # self op n/d <=> self*d op n (for n and d integers, d positive).+    # A NaN or infinity can be left unchanged without affecting the+    # comparison result.+    if isinstance(other, _numbers.Rational):+        if not self._is_special:+            self = _dec_from_triple(self._sign,+                                    str(int(self._int) * other.denominator),+                                    self._exp)+        return self, Decimal(other.numerator)++    # Comparisons with float and complex types.  == and != comparisons+    # with complex numbers should succeed, returning either True or False+    # as appropriate.  Other comparisons return NotImplemented.+    if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0:+        other = other.real+    if isinstance(other, float):+        context = getcontext()+        if equality_op:+            context.flags[FloatOperation] = 1+        else:+            context._raise_error(FloatOperation,+                "strict semantics for mixing floats and Decimals are enabled")+        return self, Decimal.from_float(other)+    return NotImplemented, NotImplemented+++##### Setup Specific Contexts ############################################++# The default context prototype used by Context()+# Is mutable, so that new contexts can have different default values++DefaultContext = Context(+        prec=28, rounding=ROUND_HALF_EVEN,+        traps=[DivisionByZero, Overflow, InvalidOperation],+        flags=[],+        Emax=999999,+        Emin=-999999,+        capitals=1,+        clamp=0+)++# Pre-made alternate contexts offered by the specification+# Don't change these; the user should be able to select these+# contexts and be able to reproduce results from other implementations+# of the spec.++BasicContext = Context(+        prec=9, rounding=ROUND_HALF_UP,+        traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow],+        flags=[],+)++ExtendedContext = Context(+        prec=9, rounding=ROUND_HALF_EVEN,+        traps=[],+        flags=[],+)+++##### crud for parsing strings #############################################+#+# Regular expression used for parsing numeric strings.  Additional+# comments:+#+# 1. Uncomment the two '\s*' lines to allow leading and/or trailing+# whitespace.  But note that the specification disallows whitespace in+# a numeric string.+#+# 2. For finite numbers (not infinities and NaNs) the body of the+# number between the optional sign and the optional exponent must have+# at least one decimal digit, possibly after the decimal point.  The+# lookahead expression '(?=\d|\.\d)' checks this.++import re+_parser = re.compile(r"""        # A numeric string consists of:+#    \s*+    (?P<sign>[-+])?              # an optional sign, followed by either...+    (+        (?=\d|\.\d)              # ...a number (with at least one digit)+        (?P<int>\d*)             # having a (possibly empty) integer part+        (\.(?P<frac>\d*))?       # followed by an optional fractional part+        (E(?P<exp>[-+]?\d+))?    # followed by an optional exponent, or...+    |+        Inf(inity)?              # ...an infinity, or...+    |+        (?P<signal>s)?           # ...an (optionally signaling)+        NaN                      # NaN+        (?P<diag>\d*)            # with (possibly empty) diagnostic info.+    )+#    \s*+    \Z+""", re.VERBOSE | re.IGNORECASE).match++_all_zeros = re.compile('0*$').match+_exact_half = re.compile('50*$').match++##### PEP3101 support functions ##############################################+# The functions in this section have little to do with the Decimal+# class, and could potentially be reused or adapted for other pure+# Python numeric classes that want to implement __format__+#+# A format specifier for Decimal looks like:+#+#   [[fill]align][sign][#][0][minimumwidth][,][.precision][type]++_parse_format_specifier_regex = re.compile(r"""\A+(?:+   (?P<fill>.)?+   (?P<align>[<>=^])+)?+(?P<sign>[-+ ])?+(?P<alt>\#)?+(?P<zeropad>0)?+(?P<minimumwidth>(?!0)\d+)?+(?P<thousands_sep>,)?+(?:\.(?P<precision>0|(?!0)\d+))?+(?P<type>[eEfFgGn%])?+\Z+""", re.VERBOSE|re.DOTALL)++del re++# The locale module is only needed for the 'n' format specifier.  The+# rest of the PEP 3101 code functions quite happily without it, so we+# don't care too much if locale isn't present.+try:+    import locale as _locale+except ImportError:+    pass++def _parse_format_specifier(format_spec, _localeconv=None):+    """Parse and validate a format specifier.++    Turns a standard numeric format specifier into a dict, with the+    following entries:++      fill: fill character to pad field to minimum width+      align: alignment type, either '<', '>', '=' or '^'+      sign: either '+', '-' or ' '+      minimumwidth: nonnegative integer giving minimum width+      zeropad: boolean, indicating whether to pad with zeros+      thousands_sep: string to use as thousands separator, or ''+      grouping: grouping for thousands separators, in format+        used by localeconv+      decimal_point: string to use for decimal point+      precision: nonnegative integer giving precision, or None+      type: one of the characters 'eEfFgG%', or None++    """+    m = _parse_format_specifier_regex.match(format_spec)+    if m is None:+        raise ValueError("Invalid format specifier: " + format_spec)++    # get the dictionary+    format_dict = m.groupdict()++    # zeropad; defaults for fill and alignment.  If zero padding+    # is requested, the fill and align fields should be absent.+    fill = format_dict['fill']+    align = format_dict['align']+    format_dict['zeropad'] = (format_dict['zeropad'] is not None)+    if format_dict['zeropad']:+        if fill is not None:+            raise ValueError("Fill character conflicts with '0'"+                             " in format specifier: " + format_spec)+        if align is not None:+            raise ValueError("Alignment conflicts with '0' in "+                             "format specifier: " + format_spec)+    format_dict['fill'] = fill or ' '+    # PEP 3101 originally specified that the default alignment should+    # be left;  it was later agreed that right-aligned makes more sense+    # for numeric types.  See http://bugs.python.org/issue6857.+    format_dict['align'] = align or '>'++    # default sign handling: '-' for negative, '' for positive+    if format_dict['sign'] is None:+        format_dict['sign'] = '-'++    # minimumwidth defaults to 0; precision remains None if not given+    format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0')+    if format_dict['precision'] is not None:+        format_dict['precision'] = int(format_dict['precision'])++    # if format type is 'g' or 'G' then a precision of 0 makes little+    # sense; convert it to 1.  Same if format type is unspecified.+    if format_dict['precision'] == 0:+        if format_dict['type'] is None or format_dict['type'] in 'gGn':+            format_dict['precision'] = 1++    # determine thousands separator, grouping, and decimal separator, and+    # add appropriate entries to format_dict+    if format_dict['type'] == 'n':+        # apart from separators, 'n' behaves just like 'g'+        format_dict['type'] = 'g'+        if _localeconv is None:+            _localeconv = _locale.localeconv()+        if format_dict['thousands_sep'] is not None:+            raise ValueError("Explicit thousands separator conflicts with "+                             "'n' type in format specifier: " + format_spec)+        format_dict['thousands_sep'] = _localeconv['thousands_sep']+        format_dict['grouping'] = _localeconv['grouping']+        format_dict['decimal_point'] = _localeconv['decimal_point']+    else:+        if format_dict['thousands_sep'] is None:+            format_dict['thousands_sep'] = ''+        format_dict['grouping'] = [3, 0]+        format_dict['decimal_point'] = '.'++    return format_dict++def _format_align(sign, body, spec):+    """Given an unpadded, non-aligned numeric string 'body' and sign+    string 'sign', add padding and alignment conforming to the given+    format specifier dictionary 'spec' (as produced by+    parse_format_specifier).++    """+    # how much extra space do we have to play with?+    minimumwidth = spec['minimumwidth']+    fill = spec['fill']+    padding = fill*(minimumwidth - len(sign) - len(body))++    align = spec['align']+    if align == '<':+        result = sign + body + padding+    elif align == '>':+        result = padding + sign + body+    elif align == '=':+        result = sign + padding + body+    elif align == '^':+        half = len(padding)//2+        result = padding[:half] + sign + body + padding[half:]+    else:+        raise ValueError('Unrecognised alignment field')++    return result++def _group_lengths(grouping):+    """Convert a localeconv-style grouping into a (possibly infinite)+    iterable of integers representing group lengths.++    """+    # The result from localeconv()['grouping'], and the input to this+    # function, should be a list of integers in one of the+    # following three forms:+    #+    #   (1) an empty list, or+    #   (2) nonempty list of positive integers + [0]+    #   (3) list of positive integers + [locale.CHAR_MAX], or++    from itertools import chain, repeat+    if not grouping:+        return []+    elif grouping[-1] == 0 and len(grouping) >= 2:+        return chain(grouping[:-1], repeat(grouping[-2]))+    elif grouping[-1] == _locale.CHAR_MAX:+        return grouping[:-1]+    else:+        raise ValueError('unrecognised format for grouping')++def _insert_thousands_sep(digits, spec, min_width=1):+    """Insert thousands separators into a digit string.++    spec is a dictionary whose keys should include 'thousands_sep' and+    'grouping'; typically it's the result of parsing the format+    specifier using _parse_format_specifier.++    The min_width keyword argument gives the minimum length of the+    result, which will be padded on the left with zeros if necessary.++    If necessary, the zero padding adds an extra '0' on the left to+    avoid a leading thousands separator.  For example, inserting+    commas every three digits in '123456', with min_width=8, gives+    '0,123,456', even though that has length 9.++    """++    sep = spec['thousands_sep']+    grouping = spec['grouping']++    groups = []+    for l in _group_lengths(grouping):+        if l <= 0:+            raise ValueError("group length should be positive")+        # max(..., 1) forces at least 1 digit to the left of a separator+        l = min(max(len(digits), min_width, 1), l)+        groups.append('0'*(l - len(digits)) + digits[-l:])+        digits = digits[:-l]+        min_width -= l+        if not digits and min_width <= 0:+            break+        min_width -= len(sep)+    else:+        l = max(len(digits), min_width, 1)+        groups.append('0'*(l - len(digits)) + digits[-l:])+    return sep.join(reversed(groups))++def _format_sign(is_negative, spec):+    """Determine sign character."""++    if is_negative:+        return '-'+    elif spec['sign'] in ' +':+        return spec['sign']+    else:+        return ''++def _format_number(is_negative, intpart, fracpart, exp, spec):+    """Format a number, given the following data:++    is_negative: true if the number is negative, else false+    intpart: string of digits that must appear before the decimal point+    fracpart: string of digits that must come after the point+    exp: exponent, as an integer+    spec: dictionary resulting from parsing the format specifier++    This function uses the information in spec to:+      insert separators (decimal separator and thousands separators)+      format the sign+      format the exponent+      add trailing '%' for the '%' type+      zero-pad if necessary+      fill and align if necessary+    """++    sign = _format_sign(is_negative, spec)++    if fracpart or spec['alt']:+        fracpart = spec['decimal_point'] + fracpart++    if exp != 0 or spec['type'] in 'eE':+        echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']]+        fracpart += "{0}{1:+}".format(echar, exp)+    if spec['type'] == '%':+        fracpart += '%'++    if spec['zeropad']:+        min_width = spec['minimumwidth'] - len(fracpart) - len(sign)+    else:+        min_width = 0+    intpart = _insert_thousands_sep(intpart, spec, min_width)++    return _format_align(sign, intpart+fracpart, spec)+++##### Useful Constants (internal use only) ################################++# Reusable defaults+_Infinity = Decimal('Inf')+_NegativeInfinity = Decimal('-Inf')+_NaN = Decimal('NaN')+_Zero = Decimal(0)+_One = Decimal(1)+_NegativeOne = Decimal(-1)++# _SignedInfinity[sign] is infinity w/ that sign+_SignedInfinity = (_Infinity, _NegativeInfinity)++# Constants related to the hash implementation;  hash(x) is based+# on the reduction of x modulo _PyHASH_MODULUS+_PyHASH_MODULUS = sys.hash_info.modulus+# hash values to use for positive and negative infinities, and nans+_PyHASH_INF = sys.hash_info.inf+_PyHASH_NAN = sys.hash_info.nan++# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS+_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)+del sys
+ test/files/pypy2.py view
@@ -0,0 +1,3134 @@++from rpython.rlib.parsing.tree import Nonterminal, Symbol+from rpython.rlib.parsing.makepackrat import PackratParser, BacktrackException, Status+++class Parser(object):+    def NAME(self):+        return self._NAME().result+    def _NAME(self):+        _key = self._pos+        _status = self._dict_NAME.get(_key, None)+        if _status is None:+            _status = self._dict_NAME[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self._regex1074651696()+            assert _status.status != _status.LEFTRECURSION+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def SPACE(self):+        return self._SPACE().result+    def _SPACE(self):+        _key = self._pos+        _status = self._dict_SPACE.get(_key, None)+        if _status is None:+            _status = self._dict_SPACE[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self.__chars__(' ')+            assert _status.status != _status.LEFTRECURSION+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def COMMENT(self):+        return self._COMMENT().result+    def _COMMENT(self):+        _key = self._pos+        _status = self._dict_COMMENT.get(_key, None)+        if _status is None:+            _status = self._dict_COMMENT[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self._regex528667127()+            assert _status.status != _status.LEFTRECURSION+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def IGNORE(self):+        return self._IGNORE().result+    def _IGNORE(self):+        _key = self._pos+        _status = self._dict_IGNORE.get(_key, None)+        if _status is None:+            _status = self._dict_IGNORE[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self._regex1979538501()+            assert _status.status != _status.LEFTRECURSION+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def newline(self):+        return self._newline().result+    def _newline(self):+        _key = self._pos+        _status = self._dict_newline.get(_key, None)+        if _status is None:+            _status = self._dict_newline[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _call_status = self._COMMENT()+                    _result = _call_status.result+                    _error = _call_status.error+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice1 = self._pos+                try:+                    _result = self._regex299149370()+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    raise BacktrackException(_error)+                _result = self._regex299149370()+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._newline()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def REGEX(self):+        return self._REGEX().result+    def _REGEX(self):+        _key = self._pos+        _status = self._dict_REGEX.get(_key, None)+        if _status is None:+            _status = self._dict_REGEX[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self._regex1006631623()+            r = _result+            _result = (Symbol('REGEX', r, None))+            assert _status.status != _status.LEFTRECURSION+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def QUOTE(self):+        return self._QUOTE().result+    def _QUOTE(self):+        _key = self._pos+        _status = self._dict_QUOTE.get(_key, None)+        if _status is None:+            _status = self._dict_QUOTE[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self._regex1124192327()+            r = _result+            _result = (Symbol('QUOTE', r, None))+            assert _status.status != _status.LEFTRECURSION+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def PYTHONCODE(self):+        return self._PYTHONCODE().result+    def _PYTHONCODE(self):+        _key = self._pos+        _status = self._dict_PYTHONCODE.get(_key, None)+        if _status is None:+            _status = self._dict_PYTHONCODE[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self._regex291086639()+            r = _result+            _result = (Symbol('PYTHONCODE', r, None))+            assert _status.status != _status.LEFTRECURSION+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def EOF(self):+        return self._EOF().result+    def _EOF(self):+        _key = self._pos+        _status = self._dict_EOF.get(_key, None)+        if _status is None:+            _status = self._dict_EOF[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _choice0 = self._pos+            _stored_result1 = _result+            try:+                _result = self.__any__()+            except BacktrackException:+                self._pos = _choice0+                _result = _stored_result1+            else:+                raise BacktrackException(None)+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._EOF()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = _exc.error+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def file(self):+        return self._file().result+    def _file(self):+        _key = self._pos+        _status = self._dict_file.get(_key, None)+        if _status is None:+            _status = self._dict_file[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _all0 = []+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._IGNORE()+                    _result = _call_status.result+                    _error = _call_status.error+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            _call_status = self._list()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            _before_discard2 = _result+            _call_status = self._EOF()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            _result = _before_discard2+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._file()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def list(self):+        return self._list().result+    def _list(self):+        _key = self._pos+        _status = self._dict_list.get(_key, None)+        if _status is None:+            _status = self._dict_list[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _all0 = []+            _call_status = self._production()+            _result = _call_status.result+            _error = _call_status.error+            _all0.append(_result)+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._production()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            content = _result+            _result = (Nonterminal('list', content))+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._list()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def production(self):+        return self._production().result+    def _production(self):+        _key = self._pos+        _status = self._dict_production.get(_key, None)+        if _status is None:+            _status = self._dict_production[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _call_status = self._NAME()+            _result = _call_status.result+            _error = _call_status.error+            name = _result+            _all0 = []+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._SPACE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            _call_status = self._productionargs()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            args = _result+            _result = self.__chars__(':')+            _all2 = []+            while 1:+                _choice3 = self._pos+                try:+                    _call_status = self._IGNORE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all2.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice3+                    break+            _result = _all2+            _call_status = self._or_()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            what = _result+            _all4 = []+            while 1:+                _choice5 = self._pos+                try:+                    _call_status = self._IGNORE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all4.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice5+                    break+            _result = _all4+            _result = self.__chars__(';')+            _all6 = []+            while 1:+                _choice7 = self._pos+                try:+                    _call_status = self._IGNORE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all6.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice7+                    break+            _result = _all6+            _result = (Nonterminal('production', [name, args, what]))+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._production()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def productionargs(self):+        return self._productionargs().result+    def _productionargs(self):+        _key = self._pos+        _status = self._dict_productionargs.get(_key, None)+        if _status is None:+            _status = self._dict_productionargs[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _result = self.__chars__('(')+                    _all1 = []+                    while 1:+                        _choice2 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = _call_status.error+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice2+                            break+                    _result = _all1+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._NAME()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _before_discard5 = _result+                            _all6 = []+                            while 1:+                                _choice7 = self._pos+                                try:+                                    _call_status = self._IGNORE()+                                    _result = _call_status.result+                                    _error = self._combine_errors(_error, _call_status.error)+                                    _all6.append(_result)+                                except BacktrackException as _exc:+                                    _error = self._combine_errors(_error, _exc.error)+                                    self._pos = _choice7+                                    break+                            _result = _all6+                            _result = self.__chars__(',')+                            _all8 = []+                            while 1:+                                _choice9 = self._pos+                                try:+                                    _call_status = self._IGNORE()+                                    _result = _call_status.result+                                    _error = self._combine_errors(_error, _call_status.error)+                                    _all8.append(_result)+                                except BacktrackException as _exc:+                                    _error = self._combine_errors(_error, _exc.error)+                                    self._pos = _choice9+                                    break+                            _result = _all8+                            _result = _before_discard5+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    args = _result+                    _call_status = self._NAME()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    arg = _result+                    _all10 = []+                    while 1:+                        _choice11 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all10.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice11+                            break+                    _result = _all10+                    _result = self.__chars__(')')+                    _all12 = []+                    while 1:+                        _choice13 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all12.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice13+                            break+                    _result = _all12+                    _result = (Nonterminal('productionargs', args + [arg]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice14 = self._pos+                try:+                    _result = (Nonterminal('productionargs', []))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice14+                    raise BacktrackException(_error)+                _result = (Nonterminal('productionargs', []))+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._productionargs()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def or_(self):+        return self._or_().result+    def _or_(self):+        _key = self._pos+        _status = self._dict_or_.get(_key, None)+        if _status is None:+            _status = self._dict_or_[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _all1 = []+                    _call_status = self._commands()+                    _result = _call_status.result+                    _error = _call_status.error+                    _before_discard2 = _result+                    _result = self.__chars__('|')+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    _result = _before_discard2+                    _all1.append(_result)+                    while 1:+                        _choice5 = self._pos+                        try:+                            _call_status = self._commands()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _before_discard6 = _result+                            _result = self.__chars__('|')+                            _all7 = []+                            while 1:+                                _choice8 = self._pos+                                try:+                                    _call_status = self._IGNORE()+                                    _result = _call_status.result+                                    _error = self._combine_errors(_error, _call_status.error)+                                    _all7.append(_result)+                                except BacktrackException as _exc:+                                    _error = self._combine_errors(_error, _exc.error)+                                    self._pos = _choice8+                                    break+                            _result = _all7+                            _result = _before_discard6+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice5+                            break+                    _result = _all1+                    l = _result+                    _call_status = self._commands()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    last = _result+                    _result = (Nonterminal('or', l + [last]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice9 = self._pos+                try:+                    _call_status = self._commands()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice9+                    raise BacktrackException(_error)+                _call_status = self._commands()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._or_()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def commands(self):+        return self._commands().result+    def _commands(self):+        _key = self._pos+        _status = self._dict_commands.get(_key, None)+        if _status is None:+            _status = self._dict_commands[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _call_status = self._command()+                    _result = _call_status.result+                    _error = _call_status.error+                    cmd = _result+                    _call_status = self._newline()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all1 = []+                    _call_status = self._command()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _before_discard2 = _result+                    _call_status = self._newline()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _result = _before_discard2+                    _all1.append(_result)+                    while 1:+                        _choice3 = self._pos+                        try:+                            _call_status = self._command()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _before_discard4 = _result+                            _call_status = self._newline()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _result = _before_discard4+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice3+                            break+                    _result = _all1+                    cmds = _result+                    _result = (Nonterminal('commands', [cmd] + cmds))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice5 = self._pos+                try:+                    _call_status = self._command()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice5+                    raise BacktrackException(_error)+                _call_status = self._command()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._commands()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def command(self):+        return self._command().result+    def _command(self):+        _key = self._pos+        _status = self._dict_command.get(_key, None)+        if _status is None:+            _status = self._dict_command[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _call_status = self._simplecommand()+            _result = _call_status.result+            _error = _call_status.error+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._command()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def simplecommand(self):+        return self._simplecommand().result+    def _simplecommand(self):+        _key = self._pos+        _status = self._dict_simplecommand.get(_key, None)+        if _status is None:+            _status = self._dict_simplecommand[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _call_status = self._return_()+                    _result = _call_status.result+                    _error = _call_status.error+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice1 = self._pos+                try:+                    _call_status = self._if_()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                _choice2 = self._pos+                try:+                    _call_status = self._named_command()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice2+                _choice3 = self._pos+                try:+                    _call_status = self._repetition()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice3+                _choice4 = self._pos+                try:+                    _call_status = self._choose()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice4+                _choice5 = self._pos+                try:+                    _call_status = self._negation()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice5+                    raise BacktrackException(_error)+                _call_status = self._negation()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._simplecommand()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def return_(self):+        return self._return_().result+    def _return_(self):+        _key = self._pos+        _status = self._dict_return_.get(_key, None)+        if _status is None:+            _status = self._dict_return_[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self.__chars__('return')+            _all0 = []+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._SPACE()+                    _result = _call_status.result+                    _error = _call_status.error+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            _call_status = self._PYTHONCODE()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            code = _result+            _all2 = []+            while 1:+                _choice3 = self._pos+                try:+                    _call_status = self._IGNORE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all2.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice3+                    break+            _result = _all2+            _result = (Nonterminal('return', [code]))+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._return_()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def if_(self):+        return self._if_().result+    def _if_(self):+        _key = self._pos+        _status = self._dict_if_.get(_key, None)+        if _status is None:+            _status = self._dict_if_[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _result = self.__chars__('do')+                    _call_status = self._newline()+                    _result = _call_status.result+                    _error = _call_status.error+                    _call_status = self._command()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    cmd = _result+                    _all1 = []+                    while 1:+                        _choice2 = self._pos+                        try:+                            _call_status = self._SPACE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice2+                            break+                    _result = _all1+                    _result = self.__chars__('if')+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._SPACE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    _call_status = self._PYTHONCODE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    condition = _result+                    _all5 = []+                    while 1:+                        _choice6 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all5.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice6+                            break+                    _result = _all5+                    _result = (Nonterminal('if', [cmd, condition]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice7 = self._pos+                try:+                    _result = self.__chars__('if')+                    _all8 = []+                    while 1:+                        _choice9 = self._pos+                        try:+                            _call_status = self._SPACE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all8.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice9+                            break+                    _result = _all8+                    _call_status = self._PYTHONCODE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    condition = _result+                    _all10 = []+                    while 1:+                        _choice11 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all10.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice11+                            break+                    _result = _all10+                    _result = (Nonterminal('if', [condition]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice7+                    raise BacktrackException(_error)+                _result = self.__chars__('if')+                _all12 = []+                while 1:+                    _choice13 = self._pos+                    try:+                        _call_status = self._SPACE()+                        _result = _call_status.result+                        _error = self._combine_errors(_error, _call_status.error)+                        _all12.append(_result)+                    except BacktrackException as _exc:+                        _error = self._combine_errors(_error, _exc.error)+                        self._pos = _choice13+                        break+                _result = _all12+                _call_status = self._PYTHONCODE()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                condition = _result+                _all14 = []+                while 1:+                    _choice15 = self._pos+                    try:+                        _call_status = self._IGNORE()+                        _result = _call_status.result+                        _error = self._combine_errors(_error, _call_status.error)+                        _all14.append(_result)+                    except BacktrackException as _exc:+                        _error = self._combine_errors(_error, _exc.error)+                        self._pos = _choice15+                        break+                _result = _all14+                _result = (Nonterminal('if', [condition]))+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._if_()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def choose(self):+        return self._choose().result+    def _choose(self):+        _key = self._pos+        _status = self._dict_choose.get(_key, None)+        if _status is None:+            _status = self._dict_choose[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _result = self.__chars__('choose')+            _all0 = []+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._SPACE()+                    _result = _call_status.result+                    _error = _call_status.error+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            _call_status = self._NAME()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            name = _result+            _all2 = []+            while 1:+                _choice3 = self._pos+                try:+                    _call_status = self._SPACE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all2.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice3+                    break+            _result = _all2+            _result = self.__chars__('in')+            _all4 = []+            while 1:+                _choice5 = self._pos+                try:+                    _call_status = self._SPACE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all4.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice5+                    break+            _result = _all4+            _call_status = self._PYTHONCODE()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            expr = _result+            _all6 = []+            while 1:+                _choice7 = self._pos+                try:+                    _call_status = self._IGNORE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all6.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice7+                    break+            _result = _all6+            _call_status = self._commands()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            cmds = _result+            _result = (Nonterminal('choose', [name, expr, cmds]))+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._choose()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def commandchain(self):+        return self._commandchain().result+    def _commandchain(self):+        _key = self._pos+        _status = self._dict_commandchain.get(_key, None)+        if _status is None:+            _status = self._dict_commandchain[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _all0 = []+            _call_status = self._simplecommand()+            _result = _call_status.result+            _error = _call_status.error+            _all0.append(_result)+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._simplecommand()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            result = _result+            _result = (Nonterminal('commands', result))+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._commandchain()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def named_command(self):+        return self._named_command().result+    def _named_command(self):+        _key = self._pos+        _status = self._dict_named_command.get(_key, None)+        if _status is None:+            _status = self._dict_named_command[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _call_status = self._NAME()+            _result = _call_status.result+            _error = _call_status.error+            name = _result+            _all0 = []+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._SPACE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            _result = self.__chars__('=')+            _all2 = []+            while 1:+                _choice3 = self._pos+                try:+                    _call_status = self._SPACE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all2.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice3+                    break+            _result = _all2+            _call_status = self._command()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            cmd = _result+            _result = (Nonterminal('named_command', [name, cmd]))+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._named_command()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def repetition(self):+        return self._repetition().result+    def _repetition(self):+        _key = self._pos+        _status = self._dict_repetition.get(_key, None)+        if _status is None:+            _status = self._dict_repetition[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _call_status = self._enclosed()+                    _result = _call_status.result+                    _error = _call_status.error+                    what = _result+                    _all1 = []+                    while 1:+                        _choice2 = self._pos+                        try:+                            _call_status = self._SPACE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice2+                            break+                    _result = _all1+                    _result = self.__chars__('?')+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    _result = (Nonterminal('maybe', [what]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice5 = self._pos+                try:+                    _call_status = self._enclosed()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    what = _result+                    _all6 = []+                    while 1:+                        _choice7 = self._pos+                        try:+                            _call_status = self._SPACE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all6.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice7+                            break+                    _result = _all6+                    while 1:+                        _choice8 = self._pos+                        try:+                            _result = self.__chars__('*')+                            break+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice8+                        _choice9 = self._pos+                        try:+                            _result = self.__chars__('+')+                            break+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice9+                            raise BacktrackException(_error)+                        _result = self.__chars__('+')+                        break+                    repetition = _result+                    _all10 = []+                    while 1:+                        _choice11 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all10.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice11+                            break+                    _result = _all10+                    _result = (Nonterminal('repetition', [repetition, what]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice5+                    raise BacktrackException(_error)+                _call_status = self._enclosed()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                what = _result+                _all12 = []+                while 1:+                    _choice13 = self._pos+                    try:+                        _call_status = self._SPACE()+                        _result = _call_status.result+                        _error = self._combine_errors(_error, _call_status.error)+                        _all12.append(_result)+                    except BacktrackException as _exc:+                        _error = self._combine_errors(_error, _exc.error)+                        self._pos = _choice13+                        break+                _result = _all12+                while 1:+                    _choice14 = self._pos+                    try:+                        _result = self.__chars__('*')+                        break+                    except BacktrackException as _exc:+                        _error = self._combine_errors(_error, _exc.error)+                        self._pos = _choice14+                    _choice15 = self._pos+                    try:+                        _result = self.__chars__('+')+                        break+                    except BacktrackException as _exc:+                        _error = self._combine_errors(_error, _exc.error)+                        self._pos = _choice15+                        raise BacktrackException(_error)+                    _result = self.__chars__('+')+                    break+                repetition = _result+                _all16 = []+                while 1:+                    _choice17 = self._pos+                    try:+                        _call_status = self._IGNORE()+                        _result = _call_status.result+                        _error = self._combine_errors(_error, _call_status.error)+                        _all16.append(_result)+                    except BacktrackException as _exc:+                        _error = self._combine_errors(_error, _exc.error)+                        self._pos = _choice17+                        break+                _result = _all16+                _result = (Nonterminal('repetition', [repetition, what]))+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._repetition()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def negation(self):+        return self._negation().result+    def _negation(self):+        _key = self._pos+        _status = self._dict_negation.get(_key, None)+        if _status is None:+            _status = self._dict_negation[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _result = self.__chars__('!')+                    _all1 = []+                    while 1:+                        _choice2 = self._pos+                        try:+                            _call_status = self._SPACE()+                            _result = _call_status.result+                            _error = _call_status.error+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice2+                            break+                    _result = _all1+                    _call_status = self._negation()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    what = _result+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    _result = (Nonterminal('negation', [what]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice5 = self._pos+                try:+                    _call_status = self._enclosed()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice5+                    raise BacktrackException(_error)+                _call_status = self._enclosed()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._negation()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def enclosed(self):+        return self._enclosed().result+    def _enclosed(self):+        _key = self._pos+        _status = self._dict_enclosed.get(_key, None)+        if _status is None:+            _status = self._dict_enclosed[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _result = self.__chars__('<')+                    _all1 = []+                    while 1:+                        _choice2 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = _call_status.error+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice2+                            break+                    _result = _all1+                    _call_status = self._primary()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    what = _result+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    _result = self.__chars__('>')+                    _all5 = []+                    while 1:+                        _choice6 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all5.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice6+                            break+                    _result = _all5+                    _result = (Nonterminal('exclusive', [what]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice7 = self._pos+                try:+                    _result = self.__chars__('[')+                    _all8 = []+                    while 1:+                        _choice9 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all8.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice9+                            break+                    _result = _all8+                    _call_status = self._or_()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    what = _result+                    _all10 = []+                    while 1:+                        _choice11 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all10.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice11+                            break+                    _result = _all10+                    _result = self.__chars__(']')+                    _all12 = []+                    while 1:+                        _choice13 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all12.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice13+                            break+                    _result = _all12+                    _result = (Nonterminal('ignore', [what]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice7+                _choice14 = self._pos+                try:+                    _before_discard15 = _result+                    _result = self.__chars__('(')+                    _all16 = []+                    while 1:+                        _choice17 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all16.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice17+                            break+                    _result = _all16+                    _result = _before_discard15+                    _call_status = self._or_()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _before_discard18 = _result+                    _result = self.__chars__(')')+                    _all19 = []+                    while 1:+                        _choice20 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all19.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice20+                            break+                    _result = _all19+                    _result = _before_discard18+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice14+                _choice21 = self._pos+                try:+                    _call_status = self._primary()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice21+                    raise BacktrackException(_error)+                _call_status = self._primary()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._enclosed()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def primary(self):+        return self._primary().result+    def _primary(self):+        _key = self._pos+        _status = self._dict_primary.get(_key, None)+        if _status is None:+            _status = self._dict_primary[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _call_status = self._call()+                    _result = _call_status.result+                    _error = _call_status.error+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice1 = self._pos+                try:+                    _call_status = self._REGEX()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _before_discard2 = _result+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    _result = _before_discard2+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                _choice5 = self._pos+                try:+                    _call_status = self._QUOTE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _before_discard6 = _result+                    _all7 = []+                    while 1:+                        _choice8 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all7.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice8+                            break+                    _result = _all7+                    _result = _before_discard6+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice5+                    raise BacktrackException(_error)+                _call_status = self._QUOTE()+                _result = _call_status.result+                _error = self._combine_errors(_error, _call_status.error)+                _before_discard9 = _result+                _all10 = []+                while 1:+                    _choice11 = self._pos+                    try:+                        _call_status = self._IGNORE()+                        _result = _call_status.result+                        _error = self._combine_errors(_error, _call_status.error)+                        _all10.append(_result)+                    except BacktrackException as _exc:+                        _error = self._combine_errors(_error, _exc.error)+                        self._pos = _choice11+                        break+                _result = _all10+                _result = _before_discard9+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._primary()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def call(self):+        return self._call().result+    def _call(self):+        _key = self._pos+        _status = self._dict_call.get(_key, None)+        if _status is None:+            _status = self._dict_call[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            _call_status = self._NAME()+            _result = _call_status.result+            _error = _call_status.error+            x = _result+            _call_status = self._arguments()+            _result = _call_status.result+            _error = self._combine_errors(_error, _call_status.error)+            args = _result+            _all0 = []+            while 1:+                _choice1 = self._pos+                try:+                    _call_status = self._IGNORE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    _all0.append(_result)+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice1+                    break+            _result = _all0+            _result = (Nonterminal("call", [x, args]))+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._call()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def arguments(self):+        return self._arguments().result+    def _arguments(self):+        _key = self._pos+        _status = self._dict_arguments.get(_key, None)+        if _status is None:+            _status = self._dict_arguments[_key] = Status()+        else:+            _statusstatus = _status.status+            if _statusstatus == _status.NORMAL:+                self._pos = _status.pos+                return _status+            elif _statusstatus == _status.ERROR:+                raise BacktrackException(_status.error)+            elif (_statusstatus == _status.INPROGRESS or+                  _statusstatus == _status.LEFTRECURSION):+                _status.status = _status.LEFTRECURSION+                if _status.result is not None:+                    self._pos = _status.pos+                    return _status+                else:+                    raise BacktrackException(None)+            elif _statusstatus == _status.SOMESOLUTIONS:+                _status.status = _status.INPROGRESS+        _startingpos = self._pos+        try:+            _result = None+            _error = None+            while 1:+                _choice0 = self._pos+                try:+                    _result = self.__chars__('(')+                    _all1 = []+                    while 1:+                        _choice2 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = _call_status.error+                            _all1.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice2+                            break+                    _result = _all1+                    _all3 = []+                    while 1:+                        _choice4 = self._pos+                        try:+                            _call_status = self._PYTHONCODE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _before_discard5 = _result+                            _all6 = []+                            while 1:+                                _choice7 = self._pos+                                try:+                                    _call_status = self._IGNORE()+                                    _result = _call_status.result+                                    _error = self._combine_errors(_error, _call_status.error)+                                    _all6.append(_result)+                                except BacktrackException as _exc:+                                    _error = self._combine_errors(_error, _exc.error)+                                    self._pos = _choice7+                                    break+                            _result = _all6+                            _result = self.__chars__(',')+                            _all8 = []+                            while 1:+                                _choice9 = self._pos+                                try:+                                    _call_status = self._IGNORE()+                                    _result = _call_status.result+                                    _error = self._combine_errors(_error, _call_status.error)+                                    _all8.append(_result)+                                except BacktrackException as _exc:+                                    _error = self._combine_errors(_error, _exc.error)+                                    self._pos = _choice9+                                    break+                            _result = _all8+                            _result = _before_discard5+                            _all3.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice4+                            break+                    _result = _all3+                    args = _result+                    _call_status = self._PYTHONCODE()+                    _result = _call_status.result+                    _error = self._combine_errors(_error, _call_status.error)+                    last = _result+                    _result = self.__chars__(')')+                    _all10 = []+                    while 1:+                        _choice11 = self._pos+                        try:+                            _call_status = self._IGNORE()+                            _result = _call_status.result+                            _error = self._combine_errors(_error, _call_status.error)+                            _all10.append(_result)+                        except BacktrackException as _exc:+                            _error = self._combine_errors(_error, _exc.error)+                            self._pos = _choice11+                            break+                    _result = _all10+                    _result = (Nonterminal("args", args + [last]))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice0+                _choice12 = self._pos+                try:+                    _result = (Nonterminal("args", []))+                    break+                except BacktrackException as _exc:+                    _error = self._combine_errors(_error, _exc.error)+                    self._pos = _choice12+                    raise BacktrackException(_error)+                _result = (Nonterminal("args", []))+                break+            if _status.status == _status.LEFTRECURSION:+                if _status.result is not None:+                    if _status.pos >= self._pos:+                        _status.status = _status.NORMAL+                        self._pos = _status.pos+                        return _status+                _status.pos = self._pos+                _status.status = _status.SOMESOLUTIONS+                _status.result = _result+                _status.error = _error+                self._pos = _startingpos+                return self._arguments()+            _status.status = _status.NORMAL+            _status.pos = self._pos+            _status.result = _result+            _status.error = _error+            return _status+        except BacktrackException as _exc:+            _status.pos = -1+            _status.result = None+            _error = self._combine_errors(_error, _exc.error)+            _status.error = _error+            _status.status = _status.ERROR+            raise BacktrackException(_error)+    def __init__(self, inputstream):+        self._dict_NAME = {}+        self._dict_SPACE = {}+        self._dict_COMMENT = {}+        self._dict_IGNORE = {}+        self._dict_newline = {}+        self._dict_REGEX = {}+        self._dict_QUOTE = {}+        self._dict_PYTHONCODE = {}+        self._dict_EOF = {}+        self._dict_file = {}+        self._dict_list = {}+        self._dict_production = {}+        self._dict_productionargs = {}+        self._dict_or_ = {}+        self._dict_commands = {}+        self._dict_command = {}+        self._dict_simplecommand = {}+        self._dict_return_ = {}+        self._dict_if_ = {}+        self._dict_choose = {}+        self._dict_commandchain = {}+        self._dict_named_command = {}+        self._dict_repetition = {}+        self._dict_negation = {}+        self._dict_enclosed = {}+        self._dict_primary = {}+        self._dict_call = {}+        self._dict_arguments = {}+        self._pos = 0+        self._inputstream = inputstream+    def _regex299149370(self):+        _choice13 = self._pos+        _runner = self._Runner(self._inputstream, self._pos)+        _i = _runner.recognize_299149370(self._pos)+        if _runner.last_matched_state == -1:+            self._pos = _choice13+            raise BacktrackException+        _upto = _runner.last_matched_index + 1+        _pos = self._pos+        assert _pos >= 0+        assert _upto >= 0+        _result = self._inputstream[_pos: _upto]+        self._pos = _upto+        return _result+    def _regex1006631623(self):+        _choice14 = self._pos+        _runner = self._Runner(self._inputstream, self._pos)+        _i = _runner.recognize_1006631623(self._pos)+        if _runner.last_matched_state == -1:+            self._pos = _choice14+            raise BacktrackException+        _upto = _runner.last_matched_index + 1+        _pos = self._pos+        assert _pos >= 0+        assert _upto >= 0+        _result = self._inputstream[_pos: _upto]+        self._pos = _upto+        return _result+    def _regex528667127(self):+        _choice15 = self._pos+        _runner = self._Runner(self._inputstream, self._pos)+        _i = _runner.recognize_528667127(self._pos)+        if _runner.last_matched_state == -1:+            self._pos = _choice15+            raise BacktrackException+        _upto = _runner.last_matched_index + 1+        _pos = self._pos+        assert _pos >= 0+        assert _upto >= 0+        _result = self._inputstream[_pos: _upto]+        self._pos = _upto+        return _result+    def _regex291086639(self):+        _choice16 = self._pos+        _runner = self._Runner(self._inputstream, self._pos)+        _i = _runner.recognize_291086639(self._pos)+        if _runner.last_matched_state == -1:+            self._pos = _choice16+            raise BacktrackException+        _upto = _runner.last_matched_index + 1+        _pos = self._pos+        assert _pos >= 0+        assert _upto >= 0+        _result = self._inputstream[_pos: _upto]+        self._pos = _upto+        return _result+    def _regex1074651696(self):+        _choice17 = self._pos+        _runner = self._Runner(self._inputstream, self._pos)+        _i = _runner.recognize_1074651696(self._pos)+        if _runner.last_matched_state == -1:+            self._pos = _choice17+            raise BacktrackException+        _upto = _runner.last_matched_index + 1+        _pos = self._pos+        assert _pos >= 0+        assert _upto >= 0+        _result = self._inputstream[_pos: _upto]+        self._pos = _upto+        return _result+    def _regex1124192327(self):+        _choice18 = self._pos+        _runner = self._Runner(self._inputstream, self._pos)+        _i = _runner.recognize_1124192327(self._pos)+        if _runner.last_matched_state == -1:+            self._pos = _choice18+            raise BacktrackException+        _upto = _runner.last_matched_index + 1+        _pos = self._pos+        assert _pos >= 0+        assert _upto >= 0+        _result = self._inputstream[_pos: _upto]+        self._pos = _upto+        return _result+    def _regex1979538501(self):+        _choice19 = self._pos+        _runner = self._Runner(self._inputstream, self._pos)+        _i = _runner.recognize_1979538501(self._pos)+        if _runner.last_matched_state == -1:+            self._pos = _choice19+            raise BacktrackException+        _upto = _runner.last_matched_index + 1+        _pos = self._pos+        assert _pos >= 0+        assert _upto >= 0+        _result = self._inputstream[_pos: _upto]+        self._pos = _upto+        return _result+    class _Runner(object):+        def __init__(self, text, pos):+            self.text = text+            self.pos = pos+            self.last_matched_state = -1+            self.last_matched_index = -1+            self.state = -1+        def recognize_299149370(runner, i):+            #auto-generated code, don't edit+            assert i >= 0+            input = runner.text+            state = 0+            while 1:+                if state == 0:+                    runner.last_matched_index = i - 1+                    runner.last_matched_state = state+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 0+                        return i+                    if char == '\n':+                        state = 1+                    elif char == ' ':+                        state = 2+                    else:+                        break+                if state == 1:+                    runner.last_matched_index = i - 1+                    runner.last_matched_state = state+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 1+                        return i+                    if char == '\n':+                        state = 1+                        continue+                    elif char == ' ':+                        state = 1+                        continue+                    else:+                        break+                if state == 2:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 2+                        return ~i+                    if char == '\n':+                        state = 1+                        continue+                    elif char == ' ':+                        state = 2+                        continue+                    else:+                        break+                runner.last_matched_state = state+                runner.last_matched_index = i - 1+                runner.state = state+                if i == len(input):+                    return i+                else:+                    return ~i+                break+            runner.state = state+            return ~i+        def recognize_1006631623(runner, i):+            #auto-generated code, don't edit+            assert i >= 0+            input = runner.text+            state = 0+            while 1:+                if state == 0:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 0+                        return ~i+                    if char == '`':+                        state = 3+                    else:+                        break+                if state == 2:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 2+                        return ~i+                    if '\x00' <= char <= '\xff':+                        state = 3+                    else:+                        break+                if state == 3:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 3+                        return ~i+                    if char == '`':+                        state = 1+                    elif char == '\\':+                        state = 2+                        continue+                    elif ']' <= char <= '_':+                        state = 3+                        continue+                    elif '\x00' <= char <= '[':+                        state = 3+                        continue+                    elif 'a' <= char <= '\xff':+                        state = 3+                        continue+                    else:+                        break+                runner.last_matched_state = state+                runner.last_matched_index = i - 1+                runner.state = state+                if i == len(input):+                    return i+                else:+                    return ~i+                break+            runner.state = state+            return ~i+        def recognize_528667127(runner, i):+            #auto-generated code, don't edit+            assert i >= 0+            input = runner.text+            state = 0+            while 1:+                if state == 0:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 0+                        return ~i+                    if char == ' ':+                        state = 0+                        continue+                    elif char == '#':+                        state = 2+                    else:+                        break+                if state == 1:+                    runner.last_matched_index = i - 1+                    runner.last_matched_state = state+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 1+                        return i+                    if char == ' ':+                        state = 0+                        continue+                    elif char == '#':+                        state = 2+                    else:+                        break+                if state == 2:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 2+                        return ~i+                    if char == '\n':+                        state = 1+                        continue+                    elif '\x00' <= char <= '\t':+                        state = 2+                        continue+                    elif '\x0b' <= char <= '\xff':+                        state = 2+                        continue+                    else:+                        break+                runner.last_matched_state = state+                runner.last_matched_index = i - 1+                runner.state = state+                if i == len(input):+                    return i+                else:+                    return ~i+                break+            runner.state = state+            return ~i+        def recognize_291086639(runner, i):+            #auto-generated code, don't edit+            assert i >= 0+            input = runner.text+            state = 0+            while 1:+                if state == 0:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 0+                        return ~i+                    if char == '{':+                        state = 2+                    else:+                        break+                if state == 2:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 2+                        return ~i+                    if char == '}':+                        state = 1+                    elif '\x00' <= char <= '\t':+                        state = 2+                        continue+                    elif '\x0b' <= char <= '|':+                        state = 2+                        continue+                    elif '~' <= char <= '\xff':+                        state = 2+                        continue+                    else:+                        break+                runner.last_matched_state = state+                runner.last_matched_index = i - 1+                runner.state = state+                if i == len(input):+                    return i+                else:+                    return ~i+                break+            runner.state = state+            return ~i+        def recognize_1074651696(runner, i):+            #auto-generated code, don't edit+            assert i >= 0+            input = runner.text+            state = 0+            while 1:+                if state == 0:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 0+                        return ~i+                    if char == '_':+                        state = 1+                    elif 'A' <= char <= 'Z':+                        state = 1+                    elif 'a' <= char <= 'z':+                        state = 1+                    else:+                        break+                if state == 1:+                    runner.last_matched_index = i - 1+                    runner.last_matched_state = state+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 1+                        return i+                    if char == '_':+                        state = 1+                        continue+                    elif '0' <= char <= '9':+                        state = 1+                        continue+                    elif 'A' <= char <= 'Z':+                        state = 1+                        continue+                    elif 'a' <= char <= 'z':+                        state = 1+                        continue+                    else:+                        break+                runner.last_matched_state = state+                runner.last_matched_index = i - 1+                runner.state = state+                if i == len(input):+                    return i+                else:+                    return ~i+                break+            runner.state = state+            return ~i+        def recognize_1124192327(runner, i):+            #auto-generated code, don't edit+            assert i >= 0+            input = runner.text+            state = 0+            while 1:+                if state == 0:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 0+                        return ~i+                    if char == "'":+                        state = 1+                    else:+                        break+                if state == 1:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 1+                        return ~i+                    if '\x00' <= char <= '&':+                        state = 1+                        continue+                    elif '(' <= char <= '\xff':+                        state = 1+                        continue+                    elif char == "'":+                        state = 2+                    else:+                        break+                runner.last_matched_state = state+                runner.last_matched_index = i - 1+                runner.state = state+                if i == len(input):+                    return i+                else:+                    return ~i+                break+            runner.state = state+            return ~i+        def recognize_1979538501(runner, i):+            #auto-generated code, don't edit+            assert i >= 0+            input = runner.text+            state = 0+            while 1:+                if state == 0:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 0+                        return ~i+                    if char == '#':+                        state = 1+                    elif char == ' ':+                        state = 2+                    elif char == '\t':+                        state = 2+                    elif char == '\n':+                        state = 2+                    else:+                        break+                if state == 1:+                    try:+                        char = input[i]+                        i += 1+                    except IndexError:+                        runner.state = 1+                        return ~i+                    if '\x00' <= char <= '\t':+                        state = 1+                        continue+                    elif '\x0b' <= char <= '\xff':+                        state = 1+                        continue+                    elif char == '\n':+                        state = 2+                    else:+                        break+                runner.last_matched_state = state+                runner.last_matched_index = i - 1+                runner.state = state+                if i == len(input):+                    return i+                else:+                    return ~i+                break+            runner.state = state+            return ~i+class PyPackratSyntaxParser(PackratParser):+    def __init__(self, stream):+        self.init_parser(stream)+forbidden = dict.fromkeys(("__weakref__ __doc__ "+                           "__dict__ __module__").split())+initthere = "__init__" in PyPackratSyntaxParser.__dict__+for key, value in Parser.__dict__.iteritems():+    if key not in PyPackratSyntaxParser.__dict__ and key not in forbidden:+        setattr(PyPackratSyntaxParser, key, value)+PyPackratSyntaxParser.init_parser = Parser.__init__.im_func
+ test/files/regex.py view
@@ -0,0 +1,1 @@+xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')
+ test/files/requests.py view
@@ -0,0 +1,980 @@+# -*- coding: utf-8 -*-++"""+requests.utils+~~~~~~~~~~~~~~++This module provides utility functions that are used within Requests+that are also useful for external consumption.+"""++import codecs+import contextlib+import io+import os+import re+import socket+import struct+import sys+import tempfile+import warnings+import zipfile++from .__version__ import __version__+from . import certs+# to_native_string is unused here, but imported here for backwards compatibility+from ._internal_utils import to_native_string+from .compat import parse_http_list as _parse_list_header+from .compat import (+    quote, urlparse, bytes, str, OrderedDict, unquote, getproxies,+    proxy_bypass, urlunparse, basestring, integer_types, is_py3,+    proxy_bypass_environment, getproxies_environment, Mapping)+from .cookies import cookiejar_from_dict+from .structures import CaseInsensitiveDict+from .exceptions import (+    InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError)++NETRC_FILES = ('.netrc', '_netrc')++DEFAULT_CA_BUNDLE_PATH = certs.where()+++if sys.platform == 'win32':+    # provide a proxy_bypass version on Windows without DNS lookups++    def proxy_bypass_registry(host):+        try:+            if is_py3:+                import winreg+            else:+                import _winreg as winreg+        except ImportError:+            return False++        try:+            internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,+                r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')+            # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it+            proxyEnable = int(winreg.QueryValueEx(internetSettings,+                                              'ProxyEnable')[0])+            # ProxyOverride is almost always a string+            proxyOverride = winreg.QueryValueEx(internetSettings,+                                                'ProxyOverride')[0]+        except OSError:+            return False+        if not proxyEnable or not proxyOverride:+            return False++        # make a check value list from the registry entry: replace the+        # '<local>' string by the localhost entry and the corresponding+        # canonical entry.+        proxyOverride = proxyOverride.split(';')+        # now check if we match one of the registry values.+        for test in proxyOverride:+            if test == '<local>':+                if '.' not in host:+                    return True+            test = test.replace(".", r"\.")     # mask dots+            test = test.replace("*", r".*")     # change glob sequence+            test = test.replace("?", r".")      # change glob char+            if re.match(test, host, re.I):+                return True+        return False++    def proxy_bypass(host):  # noqa+        """Return True, if the host should be bypassed.++        Checks proxy settings gathered from the environment, if specified,+        or the registry.+        """+        if getproxies_environment():+            return proxy_bypass_environment(host)+        else:+            return proxy_bypass_registry(host)+++def dict_to_sequence(d):+    """Returns an internal sequence dictionary update."""++    if hasattr(d, 'items'):+        d = d.items()++    return d+++def super_len(o):+    total_length = None+    current_position = 0++    if hasattr(o, '__len__'):+        total_length = len(o)++    elif hasattr(o, 'len'):+        total_length = o.len++    elif hasattr(o, 'fileno'):+        try:+            fileno = o.fileno()+        except io.UnsupportedOperation:+            pass+        else:+            total_length = os.fstat(fileno).st_size++            # Having used fstat to determine the file length, we need to+            # confirm that this file was opened up in binary mode.+            if 'b' not in o.mode:+                warnings.warn((+                    "Requests has determined the content-length for this "+                    "request using the binary size of the file: however, the "+                    "file has been opened in text mode (i.e. without the 'b' "+                    "flag in the mode). This may lead to an incorrect "+                    "content-length. In Requests 3.0, support will be removed "+                    "for files in text mode."),+                    FileModeWarning+                )++    if hasattr(o, 'tell'):+        try:+            current_position = o.tell()+        except (OSError, IOError):+            # This can happen in some weird situations, such as when the file+            # is actually a special file descriptor like stdin. In this+            # instance, we don't know what the length is, so set it to zero and+            # let requests chunk it instead.+            if total_length is not None:+                current_position = total_length+        else:+            if hasattr(o, 'seek') and total_length is None:+                # StringIO and BytesIO have seek but no useable fileno+                try:+                    # seek to end of file+                    o.seek(0, 2)+                    total_length = o.tell()++                    # seek back to current position to support+                    # partially read file-like objects+                    o.seek(current_position or 0)+                except (OSError, IOError):+                    total_length = 0++    if total_length is None:+        total_length = 0++    return max(0, total_length - current_position)+++def get_netrc_auth(url, raise_errors=False):+    """Returns the Requests tuple auth for a given url from netrc."""++    try:+        from netrc import netrc, NetrcParseError++        netrc_path = None++        for f in NETRC_FILES:+            try:+                loc = os.path.expanduser('~/{0}'.format(f))+            except KeyError:+                # os.path.expanduser can fail when $HOME is undefined and+                # getpwuid fails. See http://bugs.python.org/issue20164 &+                # https://github.com/requests/requests/issues/1846+                return++            if os.path.exists(loc):+                netrc_path = loc+                break++        # Abort early if there isn't one.+        if netrc_path is None:+            return++        ri = urlparse(url)++        # Strip port numbers from netloc. This weird `if...encode`` dance is+        # used for Python 3.2, which doesn't support unicode literals.+        splitstr = b':'+        if isinstance(url, str):+            splitstr = splitstr.decode('ascii')+        host = ri.netloc.split(splitstr)[0]++        try:+            _netrc = netrc(netrc_path).authenticators(host)+            if _netrc:+                # Return with login / password+                login_i = (0 if _netrc[0] else 1)+                return (_netrc[login_i], _netrc[2])+        except (NetrcParseError, IOError):+            # If there was a parsing error or a permissions issue reading the file,+            # we'll just skip netrc auth unless explicitly asked to raise errors.+            if raise_errors:+                raise++    # AppEngine hackiness.+    except (ImportError, AttributeError):+        pass+++def guess_filename(obj):+    """Tries to guess the filename of the given object."""+    name = getattr(obj, 'name', None)+    if (name and isinstance(name, basestring) and name[0] != '<' and+            name[-1] != '>'):+        return os.path.basename(name)+++def extract_zipped_paths(path):+    """Replace nonexistent paths that look like they refer to a member of a zip+    archive with the location of an extracted copy of the target, or else+    just return the provided path unchanged.+    """+    if os.path.exists(path):+        # this is already a valid path, no need to do anything further+        return path++    # find the first valid part of the provided path and treat that as a zip archive+    # assume the rest of the path is the name of a member in the archive+    archive, member = os.path.split(path)+    while archive and not os.path.exists(archive):+        archive, prefix = os.path.split(archive)+        member = '/'.join([prefix, member])++    if not zipfile.is_zipfile(archive):+        return path++    zip_file = zipfile.ZipFile(archive)+    if member not in zip_file.namelist():+        return path++    # we have a valid zip archive and a valid member of that archive+    tmp = tempfile.gettempdir()+    extracted_path = os.path.join(tmp, *member.split('/'))+    if not os.path.exists(extracted_path):+        extracted_path = zip_file.extract(member, path=tmp)++    return extracted_path+++def from_key_val_list(value):+    """Take an object and test to see if it can be represented as a+    dictionary. Unless it can not be represented as such, return an+    OrderedDict, e.g.,++    ::++        >>> from_key_val_list([('key', 'val')])+        OrderedDict([('key', 'val')])+        >>> from_key_val_list('string')+        ValueError: need more than 1 value to unpack+        >>> from_key_val_list({'key': 'val'})+        OrderedDict([('key', 'val')])++    :rtype: OrderedDict+    """+    if value is None:+        return None++    if isinstance(value, (str, bytes, bool, int)):+        raise ValueError('cannot encode objects that are not 2-tuples')++    return OrderedDict(value)+++def to_key_val_list(value):+    """Take an object and test to see if it can be represented as a+    dictionary. If it can be, return a list of tuples, e.g.,++    ::++        >>> to_key_val_list([('key', 'val')])+        [('key', 'val')]+        >>> to_key_val_list({'key': 'val'})+        [('key', 'val')]+        >>> to_key_val_list('string')+        ValueError: cannot encode objects that are not 2-tuples.++    :rtype: list+    """+    if value is None:+        return None++    if isinstance(value, (str, bytes, bool, int)):+        raise ValueError('cannot encode objects that are not 2-tuples')++    if isinstance(value, Mapping):+        value = value.items()++    return list(value)+++# From mitsuhiko/werkzeug (used with permission).+def parse_list_header(value):+    """Parse lists as described by RFC 2068 Section 2.++    In particular, parse comma-separated lists where the elements of+    the list may include quoted-strings.  A quoted-string could+    contain a comma.  A non-quoted string could have quotes in the+    middle.  Quotes are removed automatically after parsing.++    It basically works like :func:`parse_set_header` just that items+    may appear multiple times and case sensitivity is preserved.++    The return value is a standard :class:`list`:++    >>> parse_list_header('token, "quoted value"')+    ['token', 'quoted value']++    To create a header from the :class:`list` again, use the+    :func:`dump_header` function.++    :param value: a string with a list header.+    :return: :class:`list`+    :rtype: list+    """+    result = []+    for item in _parse_list_header(value):+        if item[:1] == item[-1:] == '"':+            item = unquote_header_value(item[1:-1])+        result.append(item)+    return result+++# From mitsuhiko/werkzeug (used with permission).+def parse_dict_header(value):+    """Parse lists of key, value pairs as described by RFC 2068 Section 2 and+    convert them into a python dict:++    >>> d = parse_dict_header('foo="is a fish", bar="as well"')+    >>> type(d) is dict+    True+    >>> sorted(d.items())+    [('bar', 'as well'), ('foo', 'is a fish')]++    If there is no value for a key it will be `None`:++    >>> parse_dict_header('key_without_value')+    {'key_without_value': None}++    To create a header from the :class:`dict` again, use the+    :func:`dump_header` function.++    :param value: a string with a dict header.+    :return: :class:`dict`+    :rtype: dict+    """+    result = {}+    for item in _parse_list_header(value):+        if '=' not in item:+            result[item] = None+            continue+        name, value = item.split('=', 1)+        if value[:1] == value[-1:] == '"':+            value = unquote_header_value(value[1:-1])+        result[name] = value+    return result+++# From mitsuhiko/werkzeug (used with permission).+def unquote_header_value(value, is_filename=False):+    r"""Unquotes a header value.  (Reversal of :func:`quote_header_value`).+    This does not use the real unquoting but what browsers are actually+    using for quoting.++    :param value: the header value to unquote.+    :rtype: str+    """+    if value and value[0] == value[-1] == '"':+        # this is not the real unquoting, but fixing this so that the+        # RFC is met will result in bugs with internet explorer and+        # probably some other browsers as well.  IE for example is+        # uploading files with "C:\foo\bar.txt" as filename+        value = value[1:-1]++        # if this is a filename and the starting characters look like+        # a UNC path, then just return the value without quotes.  Using the+        # replace sequence below on a UNC path has the effect of turning+        # the leading double slash into a single slash and then+        # _fix_ie_filename() doesn't work correctly.  See #458.+        if not is_filename or value[:2] != '\\\\':+            return value.replace('\\\\', '\\').replace('\\"', '"')+    return value+++def dict_from_cookiejar(cj):+    """Returns a key/value dictionary from a CookieJar.++    :param cj: CookieJar object to extract cookies from.+    :rtype: dict+    """++    cookie_dict = {}++    for cookie in cj:+        cookie_dict[cookie.name] = cookie.value++    return cookie_dict+++def add_dict_to_cookiejar(cj, cookie_dict):+    """Returns a CookieJar from a key/value dictionary.++    :param cj: CookieJar to insert cookies into.+    :param cookie_dict: Dict of key/values to insert into CookieJar.+    :rtype: CookieJar+    """++    return cookiejar_from_dict(cookie_dict, cj)+++def get_encodings_from_content(content):+    """Returns encodings from given content string.++    :param content: bytestring to extract encodings from.+    """+    warnings.warn((+        'In requests 3.0, get_encodings_from_content will be removed. For '+        'more information, please see the discussion on issue #2266. (This'+        ' warning should only appear once.)'),+        DeprecationWarning)++    charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)+    pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)+    xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')++    return (charset_re.findall(content) ++            pragma_re.findall(content) ++            xml_re.findall(content))+++def _parse_content_type_header(header):+    """Returns content type and parameters from given header++    :param header: string+    :return: tuple containing content type and dictionary of+         parameters+    """++    tokens = header.split(';')+    content_type, params = tokens[0].strip(), tokens[1:]+    params_dict = {}+    items_to_strip = "\"' "++    for param in params:+        param = param.strip()+        if param:+            key, value = param, True+            index_of_equals = param.find("=")+            if index_of_equals != -1:+                key = param[:index_of_equals].strip(items_to_strip)+                value = param[index_of_equals + 1:].strip(items_to_strip)+            params_dict[key.lower()] = value+    return content_type, params_dict+++def get_encoding_from_headers(headers):+    """Returns encodings from given HTTP Header Dict.++    :param headers: dictionary to extract encoding from.+    :rtype: str+    """++    content_type = headers.get('content-type')++    if not content_type:+        return None++    content_type, params = _parse_content_type_header(content_type)++    if 'charset' in params:+        return params['charset'].strip("'\"")++    if 'text' in content_type:+        return 'ISO-8859-1'+++def stream_decode_response_unicode(iterator, r):+    """Stream decodes a iterator."""++    if r.encoding is None:+        for item in iterator:+            yield item+        return++    decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace')+    for chunk in iterator:+        rv = decoder.decode(chunk)+        if rv:+            yield rv+    rv = decoder.decode(b'', final=True)+    if rv:+        yield rv+++def iter_slices(string, slice_length):+    """Iterate over slices of a string."""+    pos = 0+    if slice_length is None or slice_length <= 0:+        slice_length = len(string)+    while pos < len(string):+        yield string[pos:pos + slice_length]+        pos += slice_length+++def get_unicode_from_response(r):+    """Returns the requested content back in unicode.++    :param r: Response object to get unicode content from.++    Tried:++    1. charset from content-type+    2. fall back and replace all unicode characters++    :rtype: str+    """+    warnings.warn((+        'In requests 3.0, get_unicode_from_response will be removed. For '+        'more information, please see the discussion on issue #2266. (This'+        ' warning should only appear once.)'),+        DeprecationWarning)++    tried_encodings = []++    # Try charset from content-type+    encoding = get_encoding_from_headers(r.headers)++    if encoding:+        try:+            return str(r.content, encoding)+        except UnicodeError:+            tried_encodings.append(encoding)++    # Fall back:+    try:+        return str(r.content, encoding, errors='replace')+    except TypeError:+        return r.content+++# The unreserved URI characters (RFC 3986)+UNRESERVED_SET = frozenset(+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~")+++def unquote_unreserved(uri):+    """Un-escape any percent-escape sequences in a URI that are unreserved+    characters. This leaves all reserved, illegal and non-ASCII bytes encoded.++    :rtype: str+    """+    parts = uri.split('%')+    for i in range(1, len(parts)):+        h = parts[i][0:2]+        if len(h) == 2 and h.isalnum():+            try:+                c = chr(int(h, 16))+            except ValueError:+                raise InvalidURL("Invalid percent-escape sequence: '%s'" % h)++            if c in UNRESERVED_SET:+                parts[i] = c + parts[i][2:]+            else:+                parts[i] = '%' + parts[i]+        else:+            parts[i] = '%' + parts[i]+    return ''.join(parts)+++def requote_uri(uri):+    """Re-quote the given URI.++    This function passes the given URI through an unquote/quote cycle to+    ensure that it is fully and consistently quoted.++    :rtype: str+    """+    safe_with_percent = "!#$%&'()*+,/:;=?@[]~"+    safe_without_percent = "!#$&'()*+,/:;=?@[]~"+    try:+        # Unquote only the unreserved characters+        # Then quote only illegal characters (do not quote reserved,+        # unreserved, or '%')+        return quote(unquote_unreserved(uri), safe=safe_with_percent)+    except InvalidURL:+        # We couldn't unquote the given URI, so let's try quoting it, but+        # there may be unquoted '%'s in the URI. We need to make sure they're+        # properly quoted so they do not cause issues elsewhere.+        return quote(uri, safe=safe_without_percent)+++def address_in_network(ip, net):+    """This function allows you to check if an IP belongs to a network subnet++    Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24+             returns False if ip = 192.168.1.1 and net = 192.168.100.0/24++    :rtype: bool+    """+    ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0]+    netaddr, bits = net.split('/')+    netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0]+    network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask+    return (ipaddr & netmask) == (network & netmask)+++def dotted_netmask(mask):+    """Converts mask from /xx format to xxx.xxx.xxx.xxx++    Example: if mask is 24 function returns 255.255.255.0++    :rtype: str+    """+    bits = 0xffffffff ^ (1 << 32 - mask) - 1+    return socket.inet_ntoa(struct.pack('>I', bits))+++def is_ipv4_address(string_ip):+    """+    :rtype: bool+    """+    try:+        socket.inet_aton(string_ip)+    except socket.error:+        return False+    return True+++def is_valid_cidr(string_network):+    """+    Very simple check of the cidr format in no_proxy variable.++    :rtype: bool+    """+    if string_network.count('/') == 1:+        try:+            mask = int(string_network.split('/')[1])+        except ValueError:+            return False++        if mask < 1 or mask > 32:+            return False++        try:+            socket.inet_aton(string_network.split('/')[0])+        except socket.error:+            return False+    else:+        return False+    return True+++@contextlib.contextmanager+def set_environ(env_name, value):+    """Set the environment variable 'env_name' to 'value'++    Save previous value, yield, and then restore the previous value stored in+    the environment variable 'env_name'.++    If 'value' is None, do nothing"""+    value_changed = value is not None+    if value_changed:+        old_value = os.environ.get(env_name)+        os.environ[env_name] = value+    try:+        yield+    finally:+        if value_changed:+            if old_value is None:+                del os.environ[env_name]+            else:+                os.environ[env_name] = old_value+++def should_bypass_proxies(url, no_proxy):+    """+    Returns whether we should bypass proxies or not.++    :rtype: bool+    """+    # Prioritize lowercase environment variables over uppercase+    # to keep a consistent behaviour with other http projects (curl, wget).+    get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())++    # First check whether no_proxy is defined. If it is, check that the URL+    # we're getting isn't in the no_proxy list.+    no_proxy_arg = no_proxy+    if no_proxy is None:+        no_proxy = get_proxy('no_proxy')+    parsed = urlparse(url)++    if parsed.hostname is None:+        # URLs don't always have hostnames, e.g. file:/// urls.+        return True++    if no_proxy:+        # We need to check whether we match here. We need to see if we match+        # the end of the hostname, both with and without the port.+        no_proxy = (+            host for host in no_proxy.replace(' ', '').split(',') if host+        )++        if is_ipv4_address(parsed.hostname):+            for proxy_ip in no_proxy:+                if is_valid_cidr(proxy_ip):+                    if address_in_network(parsed.hostname, proxy_ip):+                        return True+                elif parsed.hostname == proxy_ip:+                    # If no_proxy ip was defined in plain IP notation instead of cidr notation &+                    # matches the IP of the index+                    return True+        else:+            host_with_port = parsed.hostname+            if parsed.port:+                host_with_port += ':{0}'.format(parsed.port)++            for host in no_proxy:+                if parsed.hostname.endswith(host) or host_with_port.endswith(host):+                    # The URL does match something in no_proxy, so we don't want+                    # to apply the proxies on this URL.+                    return True++    # If the system proxy settings indicate that this URL should be bypassed,+    # don't proxy.+    # The proxy_bypass function is incredibly buggy on OS X in early versions+    # of Python 2.6, so allow this call to fail. Only catch the specific+    # exceptions we've seen, though: this call failing in other ways can reveal+    # legitimate problems.+    with set_environ('no_proxy', no_proxy_arg):+        try:+            bypass = proxy_bypass(parsed.hostname)+        except (TypeError, socket.gaierror):+            bypass = False++    if bypass:+        return True++    return False+++def get_environ_proxies(url, no_proxy=None):+    """+    Return a dict of environment proxies.++    :rtype: dict+    """+    if should_bypass_proxies(url, no_proxy=no_proxy):+        return {}+    else:+        return getproxies()+++def select_proxy(url, proxies):+    """Select a proxy for the url, if applicable.++    :param url: The url being for the request+    :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs+    """+    proxies = proxies or {}+    urlparts = urlparse(url)+    if urlparts.hostname is None:+        return proxies.get(urlparts.scheme, proxies.get('all'))++    proxy_keys = [+        urlparts.scheme + '://' + urlparts.hostname,+        urlparts.scheme,+        'all://' + urlparts.hostname,+        'all',+    ]+    proxy = None+    for proxy_key in proxy_keys:+        if proxy_key in proxies:+            proxy = proxies[proxy_key]+            break++    return proxy+++def default_user_agent(name="python-requests"):+    """+    Return a string representing the default user agent.++    :rtype: str+    """+    return '%s/%s' % (name, __version__)+++def default_headers():+    """+    :rtype: requests.structures.CaseInsensitiveDict+    """+    return CaseInsensitiveDict({+        'User-Agent': default_user_agent(),+        'Accept-Encoding': ', '.join(('gzip', 'deflate')),+        'Accept': '*/*',+        'Connection': 'keep-alive',+    })+++def parse_header_links(value):+    """Return a list of parsed link headers proxies.++    i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg"++    :rtype: list+    """++    links = []++    replace_chars = ' \'"'++    value = value.strip(replace_chars)+    if not value:+        return links++    for val in re.split(', *<', value):+        try:+            url, params = val.split(';', 1)+        except ValueError:+            url, params = val, ''++        link = {'url': url.strip('<> \'"')}++        for param in params.split(';'):+            try:+                key, value = param.split('=')+            except ValueError:+                break++            link[key.strip(replace_chars)] = value.strip(replace_chars)++        links.append(link)++    return links+++# Null bytes; no need to recreate these on each call to guess_json_utf+_null = '\x00'.encode('ascii')  # encoding to ASCII for Python 3+_null2 = _null * 2+_null3 = _null * 3+++def guess_json_utf(data):+    """+    :rtype: str+    """+    # JSON always starts with two ASCII characters, so detection is as+    # easy as counting the nulls and from their location and count+    # determine the encoding. Also detect a BOM, if present.+    sample = data[:4]+    if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE):+        return 'utf-32'     # BOM included+    if sample[:3] == codecs.BOM_UTF8:+        return 'utf-8-sig'  # BOM included, MS style (discouraged)+    if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):+        return 'utf-16'     # BOM included+    nullcount = sample.count(_null)+    if nullcount == 0:+        return 'utf-8'+    if nullcount == 2:+        if sample[::2] == _null2:   # 1st and 3rd are null+            return 'utf-16-be'+        if sample[1::2] == _null2:  # 2nd and 4th are null+            return 'utf-16-le'+        # Did not detect 2 valid UTF-16 ascii-range characters+    if nullcount == 3:+        if sample[:3] == _null3:+            return 'utf-32-be'+        if sample[1:] == _null3:+            return 'utf-32-le'+        # Did not detect a valid UTF-32 ascii-range character+    return None+++def prepend_scheme_if_needed(url, new_scheme):+    """Given a URL that may or may not have a scheme, prepend the given scheme.+    Does not replace a present scheme with the one provided as an argument.++    :rtype: str+    """+    scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme)++    # urlparse is a finicky beast, and sometimes decides that there isn't a+    # netloc present. Assume that it's being over-cautious, and switch netloc+    # and path if urlparse decided there was no netloc.+    if not netloc:+        netloc, path = path, netloc++    return urlunparse((scheme, netloc, path, params, query, fragment))+++def get_auth_from_url(url):+    """Given a url with authentication components, extract them into a tuple of+    username,password.++    :rtype: (str,str)+    """+    parsed = urlparse(url)++    try:+        auth = (unquote(parsed.username), unquote(parsed.password))+    except (AttributeError, TypeError):+        auth = ('', '')++    return auth+++# Moved outside of function to avoid recompile every call+_CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$')+_CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$')+++def check_header_validity(header):+    """Verifies that header value is a string which doesn't contain+    leading whitespace or return characters. This prevents unintended+    header injection.++    :param header: tuple, in the format (name, value).+    """+    name, value = header++    if isinstance(value, bytes):+        pat = _CLEAN_HEADER_REGEX_BYTE+    else:+        pat = _CLEAN_HEADER_REGEX_STR+    try:+        if not pat.match(value):+            raise InvalidHeader("Invalid return character or leading space in header: %s" % name)+    except TypeError:+        raise InvalidHeader("Value for header {%s: %s} must be of type str or "+                            "bytes, not %s" % (name, value, type(value)))+++def urldefragauth(url):+    """+    Given a url remove the fragment and the authentication part.++    :rtype: str+    """+    scheme, netloc, path, params, query, fragment = urlparse(url)++    # see func:`prepend_scheme_if_needed`+    if not netloc:+        netloc, path = path, netloc++    netloc = netloc.rsplit('@', 1)[-1]++    return urlunparse((scheme, netloc, path, params, query, ''))+++def rewind_body(prepared_request):+    """Move file pointer back to its recorded starting position+    so it can be read again on redirect.+    """+    body_seek = getattr(prepared_request.body, 'seek', None)+    if body_seek is not None and isinstance(prepared_request._body_position, integer_types):+        try:+            body_seek(prepared_request._body_position)+        except (IOError, OSError):+            raise UnrewindableBodyError("An error occurred when rewinding request "+                                        "body for redirect.")+    else:+        raise UnrewindableBodyError("Unable to rewind request body for redirect.")
+ test/files/requests2.py view
@@ -0,0 +1,956 @@+# -*- coding: utf-8 -*-++"""+requests.models+~~~~~~~~~~~~~~~++This module contains the primary objects that power Requests.+"""++import datetime+import sys++# Import encoding now, to avoid implicit import later.+# Implicit import within threads may cause LookupError when standard library is in a ZIP,+# such as in Embedded Python. See https://github.com/requests/requests/issues/3578.+import encodings.idna++from urllib3.fields import RequestField+from urllib3.filepost import encode_multipart_formdata+from urllib3.util import parse_url+from urllib3.exceptions import (+    DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)++from io import UnsupportedOperation+from .hooks import default_hooks+from .structures import CaseInsensitiveDict++from .auth import HTTPBasicAuth+from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar+from .exceptions import (+    HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,+    ContentDecodingError, ConnectionError, StreamConsumedError)+from ._internal_utils import to_native_string, unicode_is_ascii+from .utils import (+    guess_filename, get_auth_from_url, requote_uri,+    stream_decode_response_unicode, to_key_val_list, parse_header_links,+    iter_slices, guess_json_utf, super_len, check_header_validity)+from .compat import (+    Callable, Mapping,+    cookielib, urlunparse, urlsplit, urlencode, str, bytes,+    is_py2, chardet, builtin_str, basestring)+from .compat import json as complexjson+from .status_codes import codes++#: The set of HTTP status codes that indicate an automatically+#: processable redirect.+REDIRECT_STATI = (+    codes.moved,               # 301+    codes.found,               # 302+    codes.other,               # 303+    codes.temporary_redirect,  # 307+    codes.permanent_redirect,  # 308+)++DEFAULT_REDIRECT_LIMIT = 30+CONTENT_CHUNK_SIZE = 10 * 1024+ITER_CHUNK_SIZE = 512+++class RequestEncodingMixin(object):+    @property+    def path_url(self):+        """Build the path URL to use."""++        url = []++        p = urlsplit(self.url)++        path = p.path+        if not path:+            path = '/'++        url.append(path)++        query = p.query+        if query:+            url.append('?')+            url.append(query)++        return ''.join(url)++    @staticmethod+    def _encode_params(data):+        """Encode parameters in a piece of data.++        Will successfully encode parameters when passed as a dict or a list of+        2-tuples. Order is retained if data is a list of 2-tuples but arbitrary+        if parameters are supplied as a dict.+        """++        if isinstance(data, (str, bytes)):+            return data+        elif hasattr(data, 'read'):+            return data+        elif hasattr(data, '__iter__'):+            result = []+            for k, vs in to_key_val_list(data):+                if isinstance(vs, basestring) or not hasattr(vs, '__iter__'):+                    vs = [vs]+                for v in vs:+                    if v is not None:+                        result.append(+                            (k.encode('utf-8') if isinstance(k, str) else k,+                             v.encode('utf-8') if isinstance(v, str) else v))+            return urlencode(result, doseq=True)+        else:+            return data++    @staticmethod+    def _encode_files(files, data):+        """Build the body for a multipart/form-data request.++        Will successfully encode files when passed as a dict or a list of+        tuples. Order is retained if data is a list of tuples but arbitrary+        if parameters are supplied as a dict.+        The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)+        or 4-tuples (filename, fileobj, contentype, custom_headers).+        """+        if (not files):+            raise ValueError("Files must be provided.")+        elif isinstance(data, basestring):+            raise ValueError("Data must not be a string.")++        new_fields = []+        fields = to_key_val_list(data or {})+        files = to_key_val_list(files or {})++        for field, val in fields:+            if isinstance(val, basestring) or not hasattr(val, '__iter__'):+                val = [val]+            for v in val:+                if v is not None:+                    # Don't call str() on bytestrings: in Py3 it all goes wrong.+                    if not isinstance(v, bytes):+                        v = str(v)++                    new_fields.append(+                        (field.decode('utf-8') if isinstance(field, bytes) else field,+                         v.encode('utf-8') if isinstance(v, str) else v))++        for (k, v) in files:+            # support for explicit filename+            ft = None+            fh = None+            if isinstance(v, (tuple, list)):+                if len(v) == 2:+                    fn, fp = v+                elif len(v) == 3:+                    fn, fp, ft = v+                else:+                    fn, fp, ft, fh = v+            else:+                fn = guess_filename(v) or k+                fp = v++            if isinstance(fp, (str, bytes, bytearray)):+                fdata = fp+            elif hasattr(fp, 'read'):+                fdata = fp.read()+            elif fp is None:+                continue+            else:+                fdata = fp++            rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)+            rf.make_multipart(content_type=ft)+            new_fields.append(rf)++        body, content_type = encode_multipart_formdata(new_fields)++        return body, content_type+++class RequestHooksMixin(object):+    def register_hook(self, event, hook):+        """Properly register a hook."""++        if event not in self.hooks:+            raise ValueError('Unsupported event specified, with event name "%s"' % (event))++        if isinstance(hook, Callable):+            self.hooks[event].append(hook)+        elif hasattr(hook, '__iter__'):+            self.hooks[event].extend(h for h in hook if isinstance(h, Callable))++    def deregister_hook(self, event, hook):+        """Deregister a previously registered hook.+        Returns True if the hook existed, False if not.+        """++        try:+            self.hooks[event].remove(hook)+            return True+        except ValueError:+            return False+++class Request(RequestHooksMixin):+    """A user-created :class:`Request <Request>` object.++    Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.++    :param method: HTTP method to use.+    :param url: URL to send.+    :param headers: dictionary of headers to send.+    :param files: dictionary of {filename: fileobject} files to multipart upload.+    :param data: the body to attach to the request. If a dictionary or+        list of tuples ``[(key, value)]`` is provided, form-encoding will+        take place.+    :param json: json for the body to attach to the request (if files or data is not specified).+    :param params: URL parameters to append to the URL. If a dictionary or+        list of tuples ``[(key, value)]`` is provided, form-encoding will+        take place.+    :param auth: Auth handler or (user, pass) tuple.+    :param cookies: dictionary or CookieJar of cookies to attach to this request.+    :param hooks: dictionary of callback hooks, for internal usage.++    Usage::++      >>> import requests+      >>> req = requests.Request('GET', 'http://httpbin.org/get')+      >>> req.prepare()+      <PreparedRequest [GET]>+    """++    def __init__(self,+            method=None, url=None, headers=None, files=None, data=None,+            params=None, auth=None, cookies=None, hooks=None, json=None):++        # Default empty dicts for dict params.+        data = [] if data is None else data+        files = [] if files is None else files+        headers = {} if headers is None else headers+        params = {} if params is None else params+        hooks = {} if hooks is None else hooks++        self.hooks = default_hooks()+        for (k, v) in list(hooks.items()):+            self.register_hook(event=k, hook=v)++        self.method = method+        self.url = url+        self.headers = headers+        self.files = files+        self.data = data+        self.json = json+        self.params = params+        self.auth = auth+        self.cookies = cookies++    def __repr__(self):+        return '<Request [%s]>' % (self.method)++    def prepare(self):+        """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""+        p = PreparedRequest()+        p.prepare(+            method=self.method,+            url=self.url,+            headers=self.headers,+            files=self.files,+            data=self.data,+            json=self.json,+            params=self.params,+            auth=self.auth,+            cookies=self.cookies,+            hooks=self.hooks,+        )+        return p+++class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):+    """The fully mutable :class:`PreparedRequest <PreparedRequest>` object,+    containing the exact bytes that will be sent to the server.++    Generated from either a :class:`Request <Request>` object or manually.++    Usage::++      >>> import requests+      >>> req = requests.Request('GET', 'http://httpbin.org/get')+      >>> r = req.prepare()+      <PreparedRequest [GET]>++      >>> s = requests.Session()+      >>> s.send(r)+      <Response [200]>+    """++    def __init__(self):+        #: HTTP verb to send to the server.+        self.method = None+        #: HTTP URL to send the request to.+        self.url = None+        #: dictionary of HTTP headers.+        self.headers = None+        # The `CookieJar` used to create the Cookie header will be stored here+        # after prepare_cookies is called+        self._cookies = None+        #: request body to send to the server.+        self.body = None+        #: dictionary of callback hooks, for internal usage.+        self.hooks = default_hooks()+        #: integer denoting starting position of a readable file-like body.+        self._body_position = None++    def prepare(self,+            method=None, url=None, headers=None, files=None, data=None,+            params=None, auth=None, cookies=None, hooks=None, json=None):+        """Prepares the entire request with the given parameters."""++        self.prepare_method(method)+        self.prepare_url(url, params)+        self.prepare_headers(headers)+        self.prepare_cookies(cookies)+        self.prepare_body(data, files, json)+        self.prepare_auth(auth, url)++        # Note that prepare_auth must be last to enable authentication schemes+        # such as OAuth to work on a fully prepared request.++        # This MUST go after prepare_auth. Authenticators could add a hook+        self.prepare_hooks(hooks)++    def __repr__(self):+        return '<PreparedRequest [%s]>' % (self.method)++    def copy(self):+        p = PreparedRequest()+        p.method = self.method+        p.url = self.url+        p.headers = self.headers.copy() if self.headers is not None else None+        p._cookies = _copy_cookie_jar(self._cookies)+        p.body = self.body+        p.hooks = self.hooks+        p._body_position = self._body_position+        return p++    def prepare_method(self, method):+        """Prepares the given HTTP method."""+        self.method = method+        if self.method is not None:+            self.method = to_native_string(self.method.upper())++    @staticmethod+    def _get_idna_encoded_host(host):+        import idna++        try:+            host = idna.encode(host, uts46=True).decode('utf-8')+        except idna.IDNAError:+            raise UnicodeError+        return host++    def prepare_url(self, url, params):+        """Prepares the given HTTP URL."""+        #: Accept objects that have string representations.+        #: We're unable to blindly call unicode/str functions+        #: as this will include the bytestring indicator (b'')+        #: on python 3.x.+        #: https://github.com/requests/requests/pull/2238+        if isinstance(url, bytes):+            url = url.decode('utf8')+        else:+            url = unicode(url) if is_py2 else str(url)++        # Remove leading whitespaces from url+        url = url.lstrip()++        # Don't do any URL preparation for non-HTTP schemes like `mailto`,+        # `data` etc to work around exceptions from `url_parse`, which+        # handles RFC 3986 only.+        if ':' in url and not url.lower().startswith('http'):+            self.url = url+            return++        # Support for unicode domain names and paths.+        try:+            scheme, auth, host, port, path, query, fragment = parse_url(url)+        except LocationParseError as e:+            raise InvalidURL(*e.args)++        if not scheme:+            error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?")+            error = error.format(to_native_string(url, 'utf8'))++            raise MissingSchema(error)++        if not host:+            raise InvalidURL("Invalid URL %r: No host supplied" % url)++        # In general, we want to try IDNA encoding the hostname if the string contains+        # non-ASCII characters. This allows users to automatically get the correct IDNA+        # behaviour. For strings containing only ASCII characters, we need to also verify+        # it doesn't start with a wildcard (*), before allowing the unencoded hostname.+        if not unicode_is_ascii(host):+            try:+                host = self._get_idna_encoded_host(host)+            except UnicodeError:+                raise InvalidURL('URL has an invalid label.')+        elif host.startswith(u'*'):+            raise InvalidURL('URL has an invalid label.')++        # Carefully reconstruct the network location+        netloc = auth or ''+        if netloc:+            netloc += '@'+        netloc += host+        if port:+            netloc += ':' + str(port)++        # Bare domains aren't valid URLs.+        if not path:+            path = '/'++        if is_py2:+            if isinstance(scheme, str):+                scheme = scheme.encode('utf-8')+            if isinstance(netloc, str):+                netloc = netloc.encode('utf-8')+            if isinstance(path, str):+                path = path.encode('utf-8')+            if isinstance(query, str):+                query = query.encode('utf-8')+            if isinstance(fragment, str):+                fragment = fragment.encode('utf-8')++        if isinstance(params, (str, bytes)):+            params = to_native_string(params)++        enc_params = self._encode_params(params)+        if enc_params:+            if query:+                query = '%s&%s' % (query, enc_params)+            else:+                query = enc_params++        url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))+        self.url = url++    def prepare_headers(self, headers):+        """Prepares the given HTTP headers."""++        self.headers = CaseInsensitiveDict()+        if headers:+            for header in headers.items():+                # Raise exception on invalid header value.+                check_header_validity(header)+                name, value = header+                self.headers[to_native_string(name)] = value++    def prepare_body(self, data, files, json=None):+        """Prepares the given HTTP body data."""++        # Check if file, fo, generator, iterator.+        # If not, run through normal process.++        # Nottin' on you.+        body = None+        content_type = None++        if not data and json is not None:+            # urllib3 requires a bytes-like body. Python 2's json.dumps+            # provides this natively, but Python 3 gives a Unicode string.+            content_type = 'application/json'+            body = complexjson.dumps(json)+            if not isinstance(body, bytes):+                body = body.encode('utf-8')++        is_stream = all([+            hasattr(data, '__iter__'),+            not isinstance(data, (basestring, list, tuple, Mapping))+        ])++        try:+            length = super_len(data)+        except (TypeError, AttributeError, UnsupportedOperation):+            length = None++        if is_stream:+            body = data++            if getattr(body, 'tell', None) is not None:+                # Record the current file position before reading.+                # This will allow us to rewind a file in the event+                # of a redirect.+                try:+                    self._body_position = body.tell()+                except (IOError, OSError):+                    # This differentiates from None, allowing us to catch+                    # a failed `tell()` later when trying to rewind the body+                    self._body_position = object()++            if files:+                raise NotImplementedError('Streamed bodies and files are mutually exclusive.')++            if length:+                self.headers['Content-Length'] = builtin_str(length)+            else:+                self.headers['Transfer-Encoding'] = 'chunked'+        else:+            # Multi-part file uploads.+            if files:+                (body, content_type) = self._encode_files(files, data)+            else:+                if data:+                    body = self._encode_params(data)+                    if isinstance(data, basestring) or hasattr(data, 'read'):+                        content_type = None+                    else:+                        content_type = 'application/x-www-form-urlencoded'++            self.prepare_content_length(body)++            # Add content-type if it wasn't explicitly provided.+            if content_type and ('content-type' not in self.headers):+                self.headers['Content-Type'] = content_type++        self.body = body++    def prepare_content_length(self, body):+        """Prepare Content-Length header based on request method and body"""+        if body is not None:+            length = super_len(body)+            if length:+                # If length exists, set it. Otherwise, we fallback+                # to Transfer-Encoding: chunked.+                self.headers['Content-Length'] = builtin_str(length)+        elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None:+            # Set Content-Length to 0 for methods that can have a body+            # but don't provide one. (i.e. not GET or HEAD)+            self.headers['Content-Length'] = '0'++    def prepare_auth(self, auth, url=''):+        """Prepares the given HTTP auth data."""++        # If no Auth is explicitly provided, extract it from the URL first.+        if auth is None:+            url_auth = get_auth_from_url(self.url)+            auth = url_auth if any(url_auth) else None++        if auth:+            if isinstance(auth, tuple) and len(auth) == 2:+                # special-case basic HTTP auth+                auth = HTTPBasicAuth(*auth)++            # Allow auth to make its changes.+            r = auth(self)++            # Update self to reflect the auth changes.+            self.__dict__.update(r.__dict__)++            # Recompute Content-Length+            self.prepare_content_length(self.body)++    def prepare_cookies(self, cookies):+        """Prepares the given HTTP cookie data.++        This function eventually generates a ``Cookie`` header from the+        given cookies using cookielib. Due to cookielib's design, the header+        will not be regenerated if it already exists, meaning this function+        can only be called once for the life of the+        :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls+        to ``prepare_cookies`` will have no actual effect, unless the "Cookie"+        header is removed beforehand.+        """+        if isinstance(cookies, cookielib.CookieJar):+            self._cookies = cookies+        else:+            self._cookies = cookiejar_from_dict(cookies)++        cookie_header = get_cookie_header(self._cookies, self)+        if cookie_header is not None:+            self.headers['Cookie'] = cookie_header++    def prepare_hooks(self, hooks):+        """Prepares the given hooks."""+        # hooks can be passed as None to the prepare method and to this+        # method. To prevent iterating over None, simply use an empty list+        # if hooks is False-y+        hooks = hooks or []+        for event in hooks:+            self.register_hook(event, hooks[event])+++class Response(object):+    """The :class:`Response <Response>` object, which contains a+    server's response to an HTTP request.+    """++    __attrs__ = [+        '_content', 'status_code', 'headers', 'url', 'history',+        'encoding', 'reason', 'cookies', 'elapsed', 'request'+    ]++    def __init__(self):+        self._content = False+        self._content_consumed = False+        self._next = None++        #: Integer Code of responded HTTP Status, e.g. 404 or 200.+        self.status_code = None++        #: Case-insensitive Dictionary of Response Headers.+        #: For example, ``headers['content-encoding']`` will return the+        #: value of a ``'Content-Encoding'`` response header.+        self.headers = CaseInsensitiveDict()++        #: File-like object representation of response (for advanced usage).+        #: Use of ``raw`` requires that ``stream=True`` be set on the request.+        # This requirement does not apply for use internally to Requests.+        self.raw = None++        #: Final URL location of Response.+        self.url = None++        #: Encoding to decode with when accessing r.text.+        self.encoding = None++        #: A list of :class:`Response <Response>` objects from+        #: the history of the Request. Any redirect responses will end+        #: up here. The list is sorted from the oldest to the most recent request.+        self.history = []++        #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".+        self.reason = None++        #: A CookieJar of Cookies the server sent back.+        self.cookies = cookiejar_from_dict({})++        #: The amount of time elapsed between sending the request+        #: and the arrival of the response (as a timedelta).+        #: This property specifically measures the time taken between sending+        #: the first byte of the request and finishing parsing the headers. It+        #: is therefore unaffected by consuming the response content or the+        #: value of the ``stream`` keyword argument.+        self.elapsed = datetime.timedelta(0)++        #: The :class:`PreparedRequest <PreparedRequest>` object to which this+        #: is a response.+        self.request = None++    def __enter__(self):+        return self++    def __exit__(self, *args):+        self.close()++    def __getstate__(self):+        # Consume everything; accessing the content attribute makes+        # sure the content has been fully read.+        if not self._content_consumed:+            self.content++        return dict(+            (attr, getattr(self, attr, None))+            for attr in self.__attrs__+        )++    def __setstate__(self, state):+        for name, value in state.items():+            setattr(self, name, value)++        # pickled objects do not have .raw+        setattr(self, '_content_consumed', True)+        setattr(self, 'raw', None)++    def __repr__(self):+        return '<Response [%s]>' % (self.status_code)++    def __bool__(self):+        """Returns True if :attr:`status_code` is less than 400.++        This attribute checks if the status code of the response is between+        400 and 600 to see if there was a client error or a server error. If+        the status code, is between 200 and 400, this will return True. This+        is **not** a check to see if the response code is ``200 OK``.+        """+        return self.ok++    def __nonzero__(self):+        """Returns True if :attr:`status_code` is less than 400.++        This attribute checks if the status code of the response is between+        400 and 600 to see if there was a client error or a server error. If+        the status code, is between 200 and 400, this will return True. This+        is **not** a check to see if the response code is ``200 OK``.+        """+        return self.ok++    def __iter__(self):+        """Allows you to use a response as an iterator."""+        return self.iter_content(128)++    @property+    def ok(self):+        """Returns True if :attr:`status_code` is less than 400, False if not.++        This attribute checks if the status code of the response is between+        400 and 600 to see if there was a client error or a server error. If+        the status code is between 200 and 400, this will return True. This+        is **not** a check to see if the response code is ``200 OK``.+        """+        try:+            self.raise_for_status()+        except HTTPError:+            return False+        return True++    @property+    def is_redirect(self):+        """True if this Response is a well-formed HTTP redirect that could have+        been processed automatically (by :meth:`Session.resolve_redirects`).+        """+        return ('location' in self.headers and self.status_code in REDIRECT_STATI)++    @property+    def is_permanent_redirect(self):+        """True if this Response one of the permanent versions of redirect."""+        return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))++    @property+    def next(self):+        """Returns a PreparedRequest for the next request in a redirect chain, if there is one."""+        return self._next++    @property+    def apparent_encoding(self):+        """The apparent encoding, provided by the chardet library."""+        return chardet.detect(self.content)['encoding']++    def iter_content(self, chunk_size=1, decode_unicode=False):+        """Iterates over the response data.  When stream=True is set on the+        request, this avoids reading the content at once into memory for+        large responses.  The chunk size is the number of bytes it should+        read into memory.  This is not necessarily the length of each item+        returned as decoding can take place.++        chunk_size must be of type int or None. A value of None will+        function differently depending on the value of `stream`.+        stream=True will read data as it arrives in whatever size the+        chunks are received. If stream=False, data is returned as+        a single chunk.++        If decode_unicode is True, content will be decoded using the best+        available encoding based on the response.+        """++        def generate():+            # Special case for urllib3.+            if hasattr(self.raw, 'stream'):+                try:+                    for chunk in self.raw.stream(chunk_size, decode_content=True):+                        yield chunk+                except ProtocolError as e:+                    raise ChunkedEncodingError(e)+                except DecodeError as e:+                    raise ContentDecodingError(e)+                except ReadTimeoutError as e:+                    raise ConnectionError(e)+            else:+                # Standard file-like object.+                while True:+                    chunk = self.raw.read(chunk_size)+                    if not chunk:+                        break+                    yield chunk++            self._content_consumed = True++        if self._content_consumed and isinstance(self._content, bool):+            raise StreamConsumedError()+        elif chunk_size is not None and not isinstance(chunk_size, int):+            raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size))+        # simulate reading small chunks of the content+        reused_chunks = iter_slices(self._content, chunk_size)++        stream_chunks = generate()++        chunks = reused_chunks if self._content_consumed else stream_chunks++        if decode_unicode:+            chunks = stream_decode_response_unicode(chunks, self)++        return chunks++    def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):+        """Iterates over the response data, one line at a time.  When+        stream=True is set on the request, this avoids reading the+        content at once into memory for large responses.++        .. note:: This method is not reentrant safe.+        """++        pending = None++        for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode):++            if pending is not None:+                chunk = pending + chunk++            if delimiter:+                lines = chunk.split(delimiter)+            else:+                lines = chunk.splitlines()++            if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:+                pending = lines.pop()+            else:+                pending = None++            for line in lines:+                yield line++        if pending is not None:+            yield pending++    @property+    def content(self):+        """Content of the response, in bytes."""++        if self._content is False:+            # Read the contents.+            if self._content_consumed:+                raise RuntimeError(+                    'The content for this response was already consumed')++            if self.status_code == 0 or self.raw is None:+                self._content = None+            else:+                self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''++        self._content_consumed = True+        # don't need to release the connection; that's been handled by urllib3+        # since we exhausted the data.+        return self._content++    @property+    def text(self):+        """Content of the response, in unicode.++        If Response.encoding is None, encoding will be guessed using+        ``chardet``.++        The encoding of the response content is determined based solely on HTTP+        headers, following RFC 2616 to the letter. If you can take advantage of+        non-HTTP knowledge to make a better guess at the encoding, you should+        set ``r.encoding`` appropriately before accessing this property.+        """++        # Try charset from content-type+        content = None+        encoding = self.encoding++        if not self.content:+            return str('')++        # Fallback to auto-detected encoding.+        if self.encoding is None:+            encoding = self.apparent_encoding++        # Decode unicode from given encoding.+        try:+            content = str(self.content, encoding, errors='replace')+        except (LookupError, TypeError):+            # A LookupError is raised if the encoding was not found which could+            # indicate a misspelling or similar mistake.+            #+            # A TypeError can be raised if encoding is None+            #+            # So we try blindly encoding.+            content = str(self.content, errors='replace')++        return content++    def json(self, **kwargs):+        r"""Returns the json-encoded content of a response, if any.++        :param \*\*kwargs: Optional arguments that ``json.loads`` takes.+        :raises ValueError: If the response body does not contain valid json.+        """++        if not self.encoding and self.content and len(self.content) > 3:+            # No encoding set. JSON RFC 4627 section 3 states we should expect+            # UTF-8, -16 or -32. Detect which one to use; If the detection or+            # decoding fails, fall back to `self.text` (using chardet to make+            # a best guess).+            encoding = guess_json_utf(self.content)+            if encoding is not None:+                try:+                    return complexjson.loads(+                        self.content.decode(encoding), **kwargs+                    )+                except UnicodeDecodeError:+                    # Wrong UTF codec detected; usually because it's not UTF-8+                    # but some other 8-bit codec.  This is an RFC violation,+                    # and the server didn't bother to tell us what codec *was*+                    # used.+                    pass+        return complexjson.loads(self.text, **kwargs)++    @property+    def links(self):+        """Returns the parsed header links of the response, if any."""++        header = self.headers.get('link')++        # l = MultiDict()+        l = {}++        if header:+            links = parse_header_links(header)++            for link in links:+                key = link.get('rel') or link.get('url')+                l[key] = link++        return l++    def raise_for_status(self):+        """Raises stored :class:`HTTPError`, if one occurred."""++        http_error_msg = ''+        if isinstance(self.reason, bytes):+            # We attempt to decode utf-8 first because some servers+            # choose to localize their reason strings. If the string+            # isn't utf-8, we fall back to iso-8859-1 for all other+            # encodings. (See PR #3538)+            try:+                reason = self.reason.decode('utf-8')+            except UnicodeDecodeError:+                reason = self.reason.decode('iso-8859-1')+        else:+            reason = self.reason++        if 400 <= self.status_code < 500:+            http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url)++        elif 500 <= self.status_code < 600:+            http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url)++        if http_error_msg:+            raise HTTPError(http_error_msg, response=self)++    def close(self):+        """Releases the connection back to the pool. Once this method has been+        called the underlying ``raw`` object must not be accessed again.++        *Note: Should not normally need to be called explicitly.*+        """+        if not self._content_consumed:+            self.raw.close()++        release_conn = getattr(self.raw, 'release_conn', None)+        if release_conn is not None:+            release_conn()
+ test/files/set.py view
@@ -0,0 +1,4 @@+not {+    LOOKUP_SEP.join(relation_parts),+    LOOKUP_SEP.join(relation_parts + [part])+}.isdisjoint(valid_lookups)
+ test/files/sqlalchemy.py view
@@ -0,0 +1,4440 @@+# sql/elements.py+# Copyright (C) 2005-2018 the SQLAlchemy authors and contributors+# <see AUTHORS file>+#+# This module is part of SQLAlchemy and is released under+# the MIT License: http://www.opensource.org/licenses/mit-license.php++"""Core SQL expression elements, including :class:`.ClauseElement`,+:class:`.ColumnElement`, and derived classes.++"""++from __future__ import unicode_literals++from .. import util, exc, inspection+from . import type_api+from . import operators+from .visitors import Visitable, cloned_traverse, traverse+from .annotation import Annotated+import itertools+from .base import Executable, PARSE_AUTOCOMMIT, Immutable, NO_ARG+from .base import _generative+import numbers++import re+import operator+++def _clone(element, **kw):+    return element._clone()+++def collate(expression, collation):+    """Return the clause ``expression COLLATE collation``.++    e.g.::++        collate(mycolumn, 'utf8_bin')++    produces::++        mycolumn COLLATE utf8_bin++    The collation expression is also quoted if it is a case sensitive+    identifier, e.g. contains uppercase characters.++    .. versionchanged:: 1.2 quoting is automatically applied to COLLATE+       expressions if they are case sensitive.++    """++    expr = _literal_as_binds(expression)+    return BinaryExpression(+        expr,+        CollationClause(collation),+        operators.collate, type_=expr.type)+++def between(expr, lower_bound, upper_bound, symmetric=False):+    """Produce a ``BETWEEN`` predicate clause.++    E.g.::++        from sqlalchemy import between+        stmt = select([users_table]).where(between(users_table.c.id, 5, 7))++    Would produce SQL resembling::++        SELECT id, name FROM user WHERE id BETWEEN :id_1 AND :id_2++    The :func:`.between` function is a standalone version of the+    :meth:`.ColumnElement.between` method available on all+    SQL expressions, as in::++        stmt = select([users_table]).where(users_table.c.id.between(5, 7))++    All arguments passed to :func:`.between`, including the left side+    column expression, are coerced from Python scalar values if a+    the value is not a :class:`.ColumnElement` subclass.   For example,+    three fixed values can be compared as in::++        print(between(5, 3, 7))++    Which would produce::++        :param_1 BETWEEN :param_2 AND :param_3++    :param expr: a column expression, typically a :class:`.ColumnElement`+     instance or alternatively a Python scalar expression to be coerced+     into a column expression, serving as the left side of the ``BETWEEN``+     expression.++    :param lower_bound: a column or Python scalar expression serving as the+     lower bound of the right side of the ``BETWEEN`` expression.++    :param upper_bound: a column or Python scalar expression serving as the+     upper bound of the right side of the ``BETWEEN`` expression.++    :param symmetric: if True, will render " BETWEEN SYMMETRIC ". Note+     that not all databases support this syntax.++     .. versionadded:: 0.9.5++    .. seealso::++        :meth:`.ColumnElement.between`++    """+    expr = _literal_as_binds(expr)+    return expr.between(lower_bound, upper_bound, symmetric=symmetric)+++def literal(value, type_=None):+    r"""Return a literal clause, bound to a bind parameter.++    Literal clauses are created automatically when non-+    :class:`.ClauseElement` objects (such as strings, ints, dates, etc.) are+    used in a comparison operation with a :class:`.ColumnElement` subclass,+    such as a :class:`~sqlalchemy.schema.Column` object.  Use this function+    to force the generation of a literal clause, which will be created as a+    :class:`BindParameter` with a bound value.++    :param value: the value to be bound. Can be any Python object supported by+        the underlying DB-API, or is translatable via the given type argument.++    :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` which+        will provide bind-parameter translation for this literal.++    """+    return BindParameter(None, value, type_=type_, unique=True)+++++def outparam(key, type_=None):+    """Create an 'OUT' parameter for usage in functions (stored procedures),+    for databases which support them.++    The ``outparam`` can be used like a regular function parameter.+    The "output" value will be available from the+    :class:`~sqlalchemy.engine.ResultProxy` object via its ``out_parameters``+    attribute, which returns a dictionary containing the values.++    """+    return BindParameter(+        key, None, type_=type_, unique=False, isoutparam=True)+++def not_(clause):+    """Return a negation of the given clause, i.e. ``NOT(clause)``.++    The ``~`` operator is also overloaded on all+    :class:`.ColumnElement` subclasses to produce the+    same result.++    """+    return operators.inv(_literal_as_binds(clause))+++@inspection._self_inspects+class ClauseElement(Visitable):+    """Base class for elements of a programmatically constructed SQL+    expression.++    """+    __visit_name__ = 'clause'++    _annotations = {}+    supports_execution = False+    _from_objects = []+    bind = None+    _is_clone_of = None+    is_selectable = False+    is_clause_element = True++    description = None+    _order_by_label_element = None+    _is_from_container = False++    def _clone(self):+        """Create a shallow copy of this ClauseElement.++        This method may be used by a generative API.  Its also used as+        part of the "deep" copy afforded by a traversal that combines+        the _copy_internals() method.++        """+        c = self.__class__.__new__(self.__class__)+        c.__dict__ = self.__dict__.copy()+        ClauseElement._cloned_set._reset(c)+        ColumnElement.comparator._reset(c)++        # this is a marker that helps to "equate" clauses to each other+        # when a Select returns its list of FROM clauses.  the cloning+        # process leaves around a lot of remnants of the previous clause+        # typically in the form of column expressions still attached to the+        # old table.+        c._is_clone_of = self++        return c++    @property+    def _constructor(self):+        """return the 'constructor' for this ClauseElement.++        This is for the purposes for creating a new object of+        this type.   Usually, its just the element's __class__.+        However, the "Annotated" version of the object overrides+        to return the class of its proxied element.++        """+        return self.__class__++    @util.memoized_property+    def _cloned_set(self):+        """Return the set consisting all cloned ancestors of this+        ClauseElement.++        Includes this ClauseElement.  This accessor tends to be used for+        FromClause objects to identify 'equivalent' FROM clauses, regardless+        of transformative operations.++        """+        s = util.column_set()+        f = self+        while f is not None:+            s.add(f)+            f = f._is_clone_of+        return s++    def __getstate__(self):+        d = self.__dict__.copy()+        d.pop('_is_clone_of', None)+        return d++    def _annotate(self, values):+        """return a copy of this ClauseElement with annotations+        updated by the given dictionary.++        """+        return Annotated(self, values)++    def _with_annotations(self, values):+        """return a copy of this ClauseElement with annotations+        replaced by the given dictionary.++        """+        return Annotated(self, values)++    def _deannotate(self, values=None, clone=False):+        """return a copy of this :class:`.ClauseElement` with annotations+        removed.++        :param values: optional tuple of individual values+         to remove.++        """+        if clone:+            # clone is used when we are also copying+            # the expression for a deep deannotation+            return self._clone()+        else:+            # if no clone, since we have no annotations we return+            # self+            return self++    def _execute_on_connection(self, connection, multiparams, params):+        if self.supports_execution:+            return connection._execute_clauseelement(self, multiparams, params)+        else:+            raise exc.ObjectNotExecutableError(self)++    def unique_params(self, *optionaldict, **kwargs):+        """Return a copy with :func:`bindparam()` elements replaced.++        Same functionality as ``params()``, except adds `unique=True`+        to affected bind parameters so that multiple statements can be+        used.++        """+        return self._params(True, optionaldict, kwargs)++    def params(self, *optionaldict, **kwargs):+        """Return a copy with :func:`bindparam()` elements replaced.++        Returns a copy of this ClauseElement with :func:`bindparam()`+        elements replaced with values taken from the given dictionary::++          >>> clause = column('x') + bindparam('foo')+          >>> print clause.compile().params+          {'foo':None}+          >>> print clause.params({'foo':7}).compile().params+          {'foo':7}++        """+        return self._params(False, optionaldict, kwargs)++    def _params(self, unique, optionaldict, kwargs):+        if len(optionaldict) == 1:+            kwargs.update(optionaldict[0])+        elif len(optionaldict) > 1:+            raise exc.ArgumentError(+                "params() takes zero or one positional dictionary argument")++        def visit_bindparam(bind):+            if bind.key in kwargs:+                bind.value = kwargs[bind.key]+                bind.required = False+            if unique:+                bind._convert_to_unique()+        return cloned_traverse(self, {}, {'bindparam': visit_bindparam})++    def compare(self, other, **kw):+        r"""Compare this ClauseElement to the given ClauseElement.++        Subclasses should override the default behavior, which is a+        straight identity comparison.++        \**kw are arguments consumed by subclass compare() methods and+        may be used to modify the criteria for comparison.+        (see :class:`.ColumnElement`)++        """+        return self is other++    def _copy_internals(self, clone=_clone, **kw):+        """Reassign internal elements to be clones of themselves.++        Called during a copy-and-traverse operation on newly+        shallow-copied elements to create a deep copy.++        The given clone function should be used, which may be applying+        additional transformations to the element (i.e. replacement+        traversal, cloned traversal, annotations).++        """+        pass++    def get_children(self, **kwargs):+        r"""Return immediate child elements of this :class:`.ClauseElement`.++        This is used for visit traversal.++        \**kwargs may contain flags that change the collection that is+        returned, for example to return a subset of items in order to+        cut down on larger traversals, or to return child items from a+        different context (such as schema-level collections instead of+        clause-level).++        """+        return []++    def self_group(self, against=None):+        """Apply a 'grouping' to this :class:`.ClauseElement`.++        This method is overridden by subclasses to return a+        "grouping" construct, i.e. parenthesis.   In particular+        it's used by "binary" expressions to provide a grouping+        around themselves when placed into a larger expression,+        as well as by :func:`.select` constructs when placed into+        the FROM clause of another :func:`.select`.  (Note that+        subqueries should be normally created using the+        :meth:`.Select.alias` method, as many platforms require+        nested SELECT statements to be named).++        As expressions are composed together, the application of+        :meth:`self_group` is automatic - end-user code should never+        need to use this method directly.  Note that SQLAlchemy's+        clause constructs take operator precedence into account -+        so parenthesis might not be needed, for example, in+        an expression like ``x OR (y AND z)`` - AND takes precedence+        over OR.++        The base :meth:`self_group` method of :class:`.ClauseElement`+        just returns self.+        """+        return self++    @util.dependencies("sqlalchemy.engine.default")+    def compile(self, default, bind=None, dialect=None, **kw):+        """Compile this SQL expression.++        The return value is a :class:`~.Compiled` object.+        Calling ``str()`` or ``unicode()`` on the returned value will yield a+        string representation of the result. The+        :class:`~.Compiled` object also can return a+        dictionary of bind parameter names and values+        using the ``params`` accessor.++        :param bind: An ``Engine`` or ``Connection`` from which a+            ``Compiled`` will be acquired. This argument takes precedence over+            this :class:`.ClauseElement`'s bound engine, if any.++        :param column_keys: Used for INSERT and UPDATE statements, a list of+            column names which should be present in the VALUES clause of the+            compiled statement. If ``None``, all columns from the target table+            object are rendered.++        :param dialect: A ``Dialect`` instance from which a ``Compiled``+            will be acquired. This argument takes precedence over the `bind`+            argument as well as this :class:`.ClauseElement`'s bound engine,+            if any.++        :param inline: Used for INSERT statements, for a dialect which does+            not support inline retrieval of newly generated primary key+            columns, will force the expression used to create the new primary+            key value to be rendered inline within the INSERT statement's+            VALUES clause. This typically refers to Sequence execution but may+            also refer to any server-side default generation function+            associated with a primary key `Column`.++        :param compile_kwargs: optional dictionary of additional parameters+            that will be passed through to the compiler within all "visit"+            methods.  This allows any custom flag to be passed through to+            a custom compilation construct, for example.  It is also used+            for the case of passing the ``literal_binds`` flag through::++                from sqlalchemy.sql import table, column, select++                t = table('t', column('x'))++                s = select([t]).where(t.c.x == 5)++                print s.compile(compile_kwargs={"literal_binds": True})++            .. versionadded:: 0.9.0++        .. seealso::++            :ref:`faq_sql_expression_string`++        """++        if not dialect:+            if bind:+                dialect = bind.dialect+            elif self.bind:+                dialect = self.bind.dialect+                bind = self.bind+            else:+                dialect = default.StrCompileDialect()+        return self._compiler(dialect, bind=bind, **kw)++    def _compiler(self, dialect, **kw):+        """Return a compiler appropriate for this ClauseElement, given a+        Dialect."""++        return dialect.statement_compiler(dialect, self, **kw)++    def __str__(self):+        if util.py3k:+            return str(self.compile())+        else:+            return unicode(self.compile()).encode('ascii', 'backslashreplace')++    def __and__(self, other):+        """'and' at the ClauseElement level.++        .. deprecated:: 0.9.5 - conjunctions are intended to be+           at the :class:`.ColumnElement`. level++        """+        return and_(self, other)++    def __or__(self, other):+        """'or' at the ClauseElement level.++        .. deprecated:: 0.9.5 - conjunctions are intended to be+           at the :class:`.ColumnElement`. level++        """+        return or_(self, other)++    def __invert__(self):+        if hasattr(self, 'negation_clause'):+            return self.negation_clause+        else:+            return self._negate()++    def _negate(self):+        return UnaryExpression(+            self.self_group(against=operators.inv),+            operator=operators.inv,+            negate=None)++    def __bool__(self):+        raise TypeError("Boolean value of this clause is not defined")++    __nonzero__ = __bool__++    def __repr__(self):+        friendly = self.description+        if friendly is None:+            return object.__repr__(self)+        else:+            return '<%s.%s at 0x%x; %s>' % (+                self.__module__, self.__class__.__name__, id(self), friendly)+++class ColumnElement(operators.ColumnOperators, ClauseElement):+    """Represent a column-oriented SQL expression suitable for usage in the+    "columns" clause, WHERE clause etc. of a statement.++    While the most familiar kind of :class:`.ColumnElement` is the+    :class:`.Column` object, :class:`.ColumnElement` serves as the basis+    for any unit that may be present in a SQL expression, including+    the expressions themselves, SQL functions, bound parameters,+    literal expressions, keywords such as ``NULL``, etc.+    :class:`.ColumnElement` is the ultimate base class for all such elements.++    A wide variety of SQLAlchemy Core functions work at the SQL expression+    level, and are intended to accept instances of :class:`.ColumnElement` as+    arguments.  These functions will typically document that they accept a+    "SQL expression" as an argument.  What this means in terms of SQLAlchemy+    usually refers to an input which is either already in the form of a+    :class:`.ColumnElement` object, or a value which can be **coerced** into+    one.  The coercion rules followed by most, but not all, SQLAlchemy Core+    functions with regards to SQL expressions are as follows:++        * a literal Python value, such as a string, integer or floating+          point value, boolean, datetime, ``Decimal`` object, or virtually+          any other Python object, will be coerced into a "literal bound+          value".  This generally means that a :func:`.bindparam` will be+          produced featuring the given value embedded into the construct; the+          resulting :class:`.BindParameter` object is an instance of+          :class:`.ColumnElement`.  The Python value will ultimately be sent+          to the DBAPI at execution time as a parameterized argument to the+          ``execute()`` or ``executemany()`` methods, after SQLAlchemy+          type-specific converters (e.g. those provided by any associated+          :class:`.TypeEngine` objects) are applied to the value.++        * any special object value, typically ORM-level constructs, which+          feature a method called ``__clause_element__()``.  The Core+          expression system looks for this method when an object of otherwise+          unknown type is passed to a function that is looking to coerce the+          argument into a :class:`.ColumnElement` expression.  The+          ``__clause_element__()`` method, if present, should return a+          :class:`.ColumnElement` instance.  The primary use of+          ``__clause_element__()`` within SQLAlchemy is that of class-bound+          attributes on ORM-mapped classes; a ``User`` class which contains a+          mapped attribute named ``.name`` will have a method+          ``User.name.__clause_element__()`` which when invoked returns the+          :class:`.Column` called ``name`` associated with the mapped table.++        * The Python ``None`` value is typically interpreted as ``NULL``,+          which in SQLAlchemy Core produces an instance of :func:`.null`.++    A :class:`.ColumnElement` provides the ability to generate new+    :class:`.ColumnElement`+    objects using Python expressions.  This means that Python operators+    such as ``==``, ``!=`` and ``<`` are overloaded to mimic SQL operations,+    and allow the instantiation of further :class:`.ColumnElement` instances+    which are composed from other, more fundamental :class:`.ColumnElement`+    objects.  For example, two :class:`.ColumnClause` objects can be added+    together with the addition operator ``+`` to produce+    a :class:`.BinaryExpression`.+    Both :class:`.ColumnClause` and :class:`.BinaryExpression` are subclasses+    of :class:`.ColumnElement`::++        >>> from sqlalchemy.sql import column+        >>> column('a') + column('b')+        <sqlalchemy.sql.expression.BinaryExpression object at 0x101029dd0>+        >>> print column('a') + column('b')+        a + b++    .. seealso::++        :class:`.Column`++        :func:`.expression.column`++    """++    __visit_name__ = 'column_element'+    primary_key = False+    foreign_keys = []++    _label = None+    """The named label that can be used to target+    this column in a result set.++    This label is almost always the label used when+    rendering <expr> AS <label> in a SELECT statement.  It also+    refers to a name that this column expression can be located from+    in a result set.++    For a regular Column bound to a Table, this is typically the label+    <tablename>_<columnname>.  For other constructs, different rules+    may apply, such as anonymized labels and others.++    """++    key = None+    """the 'key' that in some circumstances refers to this object in a+    Python namespace.++    This typically refers to the "key" of the column as present in the+    ``.c`` collection of a selectable, e.g. sometable.c["somekey"] would+    return a Column with a .key of "somekey".++    """++    _key_label = None+    """A label-based version of 'key' that in some circumstances refers+    to this object in a Python namespace.+++    _key_label comes into play when a select() statement is constructed with+    apply_labels(); in this case, all Column objects in the ``.c`` collection+    are rendered as <tablename>_<columnname> in SQL; this is essentially the+    value of ._label.  But to locate those columns in the ``.c`` collection,+    the name is along the lines of <tablename>_<key>; that's the typical+    value of .key_label.++    """++    _render_label_in_columns_clause = True+    """A flag used by select._columns_plus_names that helps to determine+    we are actually going to render in terms of "SELECT <col> AS <label>".+    This flag can be returned as False for some Column objects that want+    to be rendered as simple "SELECT <col>"; typically columns that don't have+    any parent table and are named the same as what the label would be+    in any case.++    """++    _resolve_label = None+    """The name that should be used to identify this ColumnElement in a+    select() object when "label resolution" logic is used; this refers+    to using a string name in an expression like order_by() or group_by()+    that wishes to target a labeled expression in the columns clause.++    The name is distinct from that of .name or ._label to account for the case+    where anonymizing logic may be used to change the name that's actually+    rendered at compile time; this attribute should hold onto the original+    name that was user-assigned when producing a .label() construct.++    """++    _allow_label_resolve = True+    """A flag that can be flipped to prevent a column from being resolvable+    by string label name."""++    _alt_names = ()++    def self_group(self, against=None):+        if (against in (operators.and_, operators.or_, operators._asbool) and+                self.type._type_affinity+                is type_api.BOOLEANTYPE._type_affinity):+            return AsBoolean(self, operators.istrue, operators.isfalse)+        elif (against in (operators.any_op, operators.all_op)):+            return Grouping(self)+        else:+            return self++    def _negate(self):+        if self.type._type_affinity is type_api.BOOLEANTYPE._type_affinity:+            # TODO: see the note in AsBoolean that it seems to assume+            # the element is the True_() / False_() constant, so this+            # is too broad+            return AsBoolean(self, operators.isfalse, operators.istrue)+        else:+            return super(ColumnElement, self)._negate()++    @util.memoized_property+    def type(self):+        return type_api.NULLTYPE++    @util.memoized_property+    def comparator(self):+        try:+            comparator_factory = self.type.comparator_factory+        except AttributeError:+            raise TypeError(+                "Object %r associated with '.type' attribute "+                "is not a TypeEngine class or object" % self.type)+        else:+            return comparator_factory(self)++    def __getattr__(self, key):+        try:+            return getattr(self.comparator, key)+        except AttributeError:+            raise AttributeError(+                'Neither %r object nor %r object has an attribute %r' % (+                    type(self).__name__,+                    type(self.comparator).__name__,+                    key)+            )++    def operate(self, op, *other, **kwargs):+        return op(self.comparator, *other, **kwargs)++    def reverse_operate(self, op, other, **kwargs):+        return op(other, self.comparator, **kwargs)++    def _bind_param(self, operator, obj, type_=None):+        return BindParameter(None, obj,+                             _compared_to_operator=operator,+                             type_=type_,+                             _compared_to_type=self.type, unique=True)++    @property+    def expression(self):+        """Return a column expression.++        Part of the inspection interface; returns self.++        """+        return self++    @property+    def _select_iterable(self):+        return (self, )++    @util.memoized_property+    def base_columns(self):+        return util.column_set(c for c in self.proxy_set+                               if not hasattr(c, '_proxies'))++    @util.memoized_property+    def proxy_set(self):+        s = util.column_set([self])+        if hasattr(self, '_proxies'):+            for c in self._proxies:+                s.update(c.proxy_set)+        return s++    def shares_lineage(self, othercolumn):+        """Return True if the given :class:`.ColumnElement`+        has a common ancestor to this :class:`.ColumnElement`."""++        return bool(self.proxy_set.intersection(othercolumn.proxy_set))++    def _compare_name_for_result(self, other):+        """Return True if the given column element compares to this one+        when targeting within a result row."""++        return hasattr(other, 'name') and hasattr(self, 'name') and \+            other.name == self.name++    def _make_proxy(+            self, selectable, name=None, name_is_truncatable=False, **kw):+        """Create a new :class:`.ColumnElement` representing this+        :class:`.ColumnElement` as it appears in the select list of a+        descending selectable.++        """+        if name is None:+            name = self.anon_label+            if self.key:+                key = self.key+            else:+                try:+                    key = str(self)+                except exc.UnsupportedCompilationError:+                    key = self.anon_label++        else:+            key = name+        co = ColumnClause(+            _as_truncated(name) if name_is_truncatable else name,+            type_=getattr(self, 'type', None),+            _selectable=selectable+        )+        co._proxies = [self]+        if selectable._is_clone_of is not None:+            co._is_clone_of = \+                selectable._is_clone_of.columns.get(key)+        selectable._columns[key] = co+        return co++    def compare(self, other, use_proxies=False, equivalents=None, **kw):+        """Compare this ColumnElement to another.++        Special arguments understood:++        :param use_proxies: when True, consider two columns that+          share a common base column as equivalent (i.e. shares_lineage())++        :param equivalents: a dictionary of columns as keys mapped to sets+          of columns. If the given "other" column is present in this+          dictionary, if any of the columns in the corresponding set() pass+          the comparison test, the result is True. This is used to expand the+          comparison to other columns that may be known to be equivalent to+          this one via foreign key or other criterion.++        """+        to_compare = (other, )+        if equivalents and other in equivalents:+            to_compare = equivalents[other].union(to_compare)++        for oth in to_compare:+            if use_proxies and self.shares_lineage(oth):+                return True+            elif hash(oth) == hash(self):+                return True+        else:+            return False++    def cast(self, type_):+        """Produce a type cast, i.e. ``CAST(<expression> AS <type>)``.++        This is a shortcut to the :func:`~.expression.cast` function.++        .. versionadded:: 1.0.7++        """+        return Cast(self, type_)++    def label(self, name):+        """Produce a column label, i.e. ``<columnname> AS <name>``.++        This is a shortcut to the :func:`~.expression.label` function.++        if 'name' is None, an anonymous label name will be generated.++        """+        return Label(name, self, self.type)++    @util.memoized_property+    def anon_label(self):+        """provides a constant 'anonymous label' for this ColumnElement.++        This is a label() expression which will be named at compile time.+        The same label() is returned each time anon_label is called so+        that expressions can reference anon_label multiple times, producing+        the same label name at compile time.++        the compiler uses this function automatically at compile time+        for expressions that are known to be 'unnamed' like binary+        expressions and function calls.++        """+        while self._is_clone_of is not None:+            self = self._is_clone_of++        return _anonymous_label(+            '%%(%d %s)s' % (id(self), getattr(self, 'name', 'anon'))+        )+++class BindParameter(ColumnElement):+    r"""Represent a "bound expression".++    :class:`.BindParameter` is invoked explicitly using the+    :func:`.bindparam` function, as in::++        from sqlalchemy import bindparam++        stmt = select([users_table]).\+                    where(users_table.c.name == bindparam('username'))++    Detailed discussion of how :class:`.BindParameter` is used is+    at :func:`.bindparam`.++    .. seealso::++        :func:`.bindparam`++    """++    __visit_name__ = 'bindparam'++    _is_crud = False++    def __init__(self, key, value=NO_ARG, type_=None,+                 unique=False, required=NO_ARG,+                 quote=None, callable_=None,+                 expanding=False,+                 isoutparam=False,+                 _compared_to_operator=None,+                 _compared_to_type=None):+        r"""Produce a "bound expression".++        The return value is an instance of :class:`.BindParameter`; this+        is a :class:`.ColumnElement` subclass which represents a so-called+        "placeholder" value in a SQL expression, the value of which is+        supplied at the point at which the statement in executed against a+        database connection.++        In SQLAlchemy, the :func:`.bindparam` construct has+        the ability to carry along the actual value that will be ultimately+        used at expression time.  In this way, it serves not just as+        a "placeholder" for eventual population, but also as a means of+        representing so-called "unsafe" values which should not be rendered+        directly in a SQL statement, but rather should be passed along+        to the :term:`DBAPI` as values which need to be correctly escaped+        and potentially handled for type-safety.++        When using :func:`.bindparam` explicitly, the use case is typically+        one of traditional deferment of parameters; the :func:`.bindparam`+        construct accepts a name which can then be referred to at execution+        time::++            from sqlalchemy import bindparam++            stmt = select([users_table]).\+                        where(users_table.c.name == bindparam('username'))++        The above statement, when rendered, will produce SQL similar to::++            SELECT id, name FROM user WHERE name = :username++        In order to populate the value of ``:username`` above, the value+        would typically be applied at execution time to a method+        like :meth:`.Connection.execute`::++            result = connection.execute(stmt, username='wendy')++        Explicit use of :func:`.bindparam` is also common when producing+        UPDATE or DELETE statements that are to be invoked multiple times,+        where the WHERE criterion of the statement is to change on each+        invocation, such as::++            stmt = (users_table.update().+                    where(user_table.c.name == bindparam('username')).+                    values(fullname=bindparam('fullname'))+                    )++            connection.execute(+                stmt, [{"username": "wendy", "fullname": "Wendy Smith"},+                       {"username": "jack", "fullname": "Jack Jones"},+                       ]+            )++        SQLAlchemy's Core expression system makes wide use of+        :func:`.bindparam` in an implicit sense.   It is typical that Python+        literal values passed to virtually all SQL expression functions are+        coerced into fixed :func:`.bindparam` constructs.  For example, given+        a comparison operation such as::++            expr = users_table.c.name == 'Wendy'++        The above expression will produce a :class:`.BinaryExpression`+        construct, where the left side is the :class:`.Column` object+        representing the ``name`` column, and the right side is a+        :class:`.BindParameter` representing the literal value::++            print(repr(expr.right))+            BindParameter('%(4327771088 name)s', 'Wendy', type_=String())++        The expression above will render SQL such as::++            user.name = :name_1++        Where the ``:name_1`` parameter name is an anonymous name.  The+        actual string ``Wendy`` is not in the rendered string, but is carried+        along where it is later used within statement execution.  If we+        invoke a statement like the following::++            stmt = select([users_table]).where(users_table.c.name == 'Wendy')+            result = connection.execute(stmt)++        We would see SQL logging output as::++            SELECT "user".id, "user".name+            FROM "user"+            WHERE "user".name = %(name_1)s+            {'name_1': 'Wendy'}++        Above, we see that ``Wendy`` is passed as a parameter to the database,+        while the placeholder ``:name_1`` is rendered in the appropriate form+        for the target database, in this case the PostgreSQL database.++        Similarly, :func:`.bindparam` is invoked automatically+        when working with :term:`CRUD` statements as far as the "VALUES"+        portion is concerned.   The :func:`.insert` construct produces an+        ``INSERT`` expression which will, at statement execution time,+        generate bound placeholders based on the arguments passed, as in::++            stmt = users_table.insert()+            result = connection.execute(stmt, name='Wendy')++        The above will produce SQL output as::++            INSERT INTO "user" (name) VALUES (%(name)s)+            {'name': 'Wendy'}++        The :class:`.Insert` construct, at compilation/execution time,+        rendered a single :func:`.bindparam` mirroring the column+        name ``name`` as a result of the single ``name`` parameter+        we passed to the :meth:`.Connection.execute` method.++        :param key:+          the key (e.g. the name) for this bind param.+          Will be used in the generated+          SQL statement for dialects that use named parameters.  This+          value may be modified when part of a compilation operation,+          if other :class:`BindParameter` objects exist with the same+          key, or if its length is too long and truncation is+          required.++        :param value:+          Initial value for this bind param.  Will be used at statement+          execution time as the value for this parameter passed to the+          DBAPI, if no other value is indicated to the statement execution+          method for this particular parameter name.  Defaults to ``None``.++        :param callable\_:+          A callable function that takes the place of "value".  The function+          will be called at statement execution time to determine the+          ultimate value.   Used for scenarios where the actual bind+          value cannot be determined at the point at which the clause+          construct is created, but embedded bind values are still desirable.++        :param type\_:+          A :class:`.TypeEngine` class or instance representing an optional+          datatype for this :func:`.bindparam`.  If not passed, a type+          may be determined automatically for the bind, based on the given+          value; for example, trivial Python types such as ``str``,+          ``int``, ``bool``+          may result in the :class:`.String`, :class:`.Integer` or+          :class:`.Boolean` types being automatically selected.++          The type of a :func:`.bindparam` is significant especially in that+          the type will apply pre-processing to the value before it is+          passed to the database.  For example, a :func:`.bindparam` which+          refers to a datetime value, and is specified as holding the+          :class:`.DateTime` type, may apply conversion needed to the+          value (such as stringification on SQLite) before passing the value+          to the database.++        :param unique:+          if True, the key name of this :class:`.BindParameter` will be+          modified if another :class:`.BindParameter` of the same name+          already has been located within the containing+          expression.  This flag is used generally by the internals+          when producing so-called "anonymous" bound expressions, it+          isn't generally applicable to explicitly-named :func:`.bindparam`+          constructs.++        :param required:+          If ``True``, a value is required at execution time.  If not passed,+          it defaults to ``True`` if neither :paramref:`.bindparam.value`+          or :paramref:`.bindparam.callable` were passed.  If either of these+          parameters are present, then :paramref:`.bindparam.required`+          defaults to ``False``.++          .. versionchanged:: 0.8 If the ``required`` flag is not specified,+             it will be set automatically to ``True`` or ``False`` depending+             on whether or not the ``value`` or ``callable`` parameters+             were specified.++        :param quote:+          True if this parameter name requires quoting and is not+          currently known as a SQLAlchemy reserved word; this currently+          only applies to the Oracle backend, where bound names must+          sometimes be quoted.++        :param isoutparam:+          if True, the parameter should be treated like a stored procedure+          "OUT" parameter.  This applies to backends such as Oracle which+          support OUT parameters.++        :param expanding:+          if True, this parameter will be treated as an "expanding" parameter+          at execution time; the parameter value is expected to be a sequence,+          rather than a scalar value, and the string SQL statement will+          be transformed on a per-execution basis to accomodate the sequence+          with a variable number of parameter slots passed to the DBAPI.+          This is to allow statement caching to be used in conjunction with+          an IN clause.++          .. note:: The "expanding" feature does not support "executemany"-+             style parameter sets, nor does it support empty IN expressions.++          .. note:: The "expanding" feature should be considered as+             **experimental** within the 1.2 series.++          .. versionadded:: 1.2++        .. seealso::++            :ref:`coretutorial_bind_param`++            :ref:`coretutorial_insert_expressions`++            :func:`.outparam`++        """+        if isinstance(key, ColumnClause):+            type_ = key.type+            key = key.key+        if required is NO_ARG:+            required = (value is NO_ARG and callable_ is None)+        if value is NO_ARG:+            value = None++        if quote is not None:+            key = quoted_name(key, quote)++        if unique:+            self.key = _anonymous_label('%%(%d %s)s' % (id(self), key+                                                        or 'param'))+        else:+            self.key = key or _anonymous_label('%%(%d param)s'+                                               % id(self))++        # identifying key that won't change across+        # clones, used to identify the bind's logical+        # identity+        self._identifying_key = self.key++        # key that was passed in the first place, used to+        # generate new keys+        self._orig_key = key or 'param'++        self.unique = unique+        self.value = value+        self.callable = callable_+        self.isoutparam = isoutparam+        self.required = required+        self.expanding = expanding++        if type_ is None:+            if _compared_to_type is not None:+                self.type = \+                    _compared_to_type.coerce_compared_value(+                        _compared_to_operator, value)+            else:+                self.type = type_api._resolve_value_to_type(value)+        elif isinstance(type_, type):+            self.type = type_()+        else:+            self.type = type_++    def _with_value(self, value):+        """Return a copy of this :class:`.BindParameter` with the given value+        set.+        """+        cloned = self._clone()+        cloned.value = value+        cloned.callable = None+        cloned.required = False+        if cloned.type is type_api.NULLTYPE:+            cloned.type = type_api._resolve_value_to_type(value)+        return cloned++    @property+    def effective_value(self):+        """Return the value of this bound parameter,+        taking into account if the ``callable`` parameter+        was set.++        The ``callable`` value will be evaluated+        and returned if present, else ``value``.++        """+        if self.callable:+            return self.callable()+        else:+            return self.value++    def _clone(self):+        c = ClauseElement._clone(self)+        if self.unique:+            c.key = _anonymous_label('%%(%d %s)s' % (id(c), c._orig_key+                                                     or 'param'))+        return c++    def _convert_to_unique(self):+        if not self.unique:+            self.unique = True+            self.key = _anonymous_label(+                '%%(%d %s)s' % (id(self), self._orig_key or 'param'))++    def compare(self, other, **kw):+        """Compare this :class:`BindParameter` to the given+        clause."""++        return isinstance(other, BindParameter) \+            and self.type._compare_type_affinity(other.type) \+            and self.value == other.value \+            and self.callable == other.callable++    def __getstate__(self):+        """execute a deferred value for serialization purposes."""++        d = self.__dict__.copy()+        v = self.value+        if self.callable:+            v = self.callable()+            d['callable'] = None+        d['value'] = v+        return d++    def __repr__(self):+        return 'BindParameter(%r, %r, type_=%r)' % (self.key,+                                                    self.value, self.type)+++class TypeClause(ClauseElement):+    """Handle a type keyword in a SQL statement.++    Used by the ``Case`` statement.++    """++    __visit_name__ = 'typeclause'++    def __init__(self, type):+        self.type = type+++class TextClause(Executable, ClauseElement):+    """Represent a literal SQL text fragment.++    E.g.::++        from sqlalchemy import text++        t = text("SELECT * FROM users")+        result = connection.execute(t)+++    The :class:`.Text` construct is produced using the :func:`.text`+    function; see that function for full documentation.++    .. seealso::++        :func:`.text`++    """++    __visit_name__ = 'textclause'++    _bind_params_regex = re.compile(r'(?<![:\w\x5c]):(\w+)(?!:)', re.UNICODE)+    _execution_options = \+        Executable._execution_options.union(+            {'autocommit': PARSE_AUTOCOMMIT})++    @property+    def _select_iterable(self):+        return (self,)++    @property+    def selectable(self):+        # allows text() to be considered by+        # _interpret_as_from+        return self++    _hide_froms = []++    # help in those cases where text() is+    # interpreted in a column expression situation+    key = _label = _resolve_label = None++    _allow_label_resolve = False++    def __init__(+            self,+            text,+            bind=None):+        self._bind = bind+        self._bindparams = {}++        def repl(m):+            self._bindparams[m.group(1)] = BindParameter(m.group(1))+            return ':%s' % m.group(1)++        # scan the string and search for bind parameter names, add them+        # to the list of bindparams+        self.text = self._bind_params_regex.sub(repl, text)++    @classmethod+    def _create_text(self, text, bind=None, bindparams=None,+                     typemap=None, autocommit=None):+        r"""Construct a new :class:`.TextClause` clause, representing+        a textual SQL string directly.++        E.g.::++            from sqlalchemy import text++            t = text("SELECT * FROM users")+            result = connection.execute(t)++        The advantages :func:`.text` provides over a plain string are+        backend-neutral support for bind parameters, per-statement+        execution options, as well as+        bind parameter and result-column typing behavior, allowing+        SQLAlchemy type constructs to play a role when executing+        a statement that is specified literally.  The construct can also+        be provided with a ``.c`` collection of column elements, allowing+        it to be embedded in other SQL expression constructs as a subquery.++        Bind parameters are specified by name, using the format ``:name``.+        E.g.::++            t = text("SELECT * FROM users WHERE id=:user_id")+            result = connection.execute(t, user_id=12)++        For SQL statements where a colon is required verbatim, as within+        an inline string, use a backslash to escape::++            t = text("SELECT * FROM users WHERE name='\:username'")++        The :class:`.TextClause` construct includes methods which can+        provide information about the bound parameters as well as the column+        values which would be returned from the textual statement, assuming+        it's an executable SELECT type of statement.  The+        :meth:`.TextClause.bindparams` method is used to provide bound+        parameter detail, and :meth:`.TextClause.columns` method allows+        specification of return columns including names and types::++            t = text("SELECT * FROM users WHERE id=:user_id").\+                    bindparams(user_id=7).\+                    columns(id=Integer, name=String)++            for id, name in connection.execute(t):+                print(id, name)++        The :func:`.text` construct is used in cases when+        a literal string SQL fragment is specified as part of a larger query,+        such as for the WHERE clause of a SELECT statement::++            s = select([users.c.id, users.c.name]).where(text("id=:user_id"))+            result = connection.execute(s, user_id=12)++        :func:`.text` is also used for the construction+        of a full, standalone statement using plain text.+        As such, SQLAlchemy refers+        to it as an :class:`.Executable` object, and it supports+        the :meth:`Executable.execution_options` method.  For example,+        a :func:`.text` construct that should be subject to "autocommit"+        can be set explicitly so using the+        :paramref:`.Connection.execution_options.autocommit` option::++            t = text("EXEC my_procedural_thing()").\+                    execution_options(autocommit=True)++        Note that SQLAlchemy's usual "autocommit" behavior applies to+        :func:`.text` constructs implicitly - that is, statements which begin+        with a phrase such as ``INSERT``, ``UPDATE``, ``DELETE``,+        or a variety of other phrases specific to certain backends, will+        be eligible for autocommit if no transaction is in progress.++        :param text:+          the text of the SQL statement to be created.  use ``:<param>``+          to specify bind parameters; they will be compiled to their+          engine-specific format.++        :param autocommit:+          Deprecated.  Use .execution_options(autocommit=<True|False>)+          to set the autocommit option.++        :param bind:+          an optional connection or engine to be used for this text query.++        :param bindparams:+          Deprecated.  A list of :func:`.bindparam` instances used to+          provide information about parameters embedded in the statement.+          This argument now invokes the :meth:`.TextClause.bindparams`+          method on the construct before returning it.  E.g.::++              stmt = text("SELECT * FROM table WHERE id=:id",+                        bindparams=[bindparam('id', value=5, type_=Integer)])++          Is equivalent to::++              stmt = text("SELECT * FROM table WHERE id=:id").\+                        bindparams(bindparam('id', value=5, type_=Integer))++          .. deprecated:: 0.9.0 the :meth:`.TextClause.bindparams` method+             supersedes the ``bindparams`` argument to :func:`.text`.++        :param typemap:+          Deprecated.  A dictionary mapping the names of columns+          represented in the columns clause of a ``SELECT`` statement+          to type objects,+          which will be used to perform post-processing on columns within+          the result set.  This parameter now invokes the+          :meth:`.TextClause.columns` method, which returns a+          :class:`.TextAsFrom` construct that gains a ``.c`` collection and+          can be embedded in other expressions.  E.g.::++              stmt = text("SELECT * FROM table",+                            typemap={'id': Integer, 'name': String},+                        )++          Is equivalent to::++              stmt = text("SELECT * FROM table").columns(id=Integer,+                                                         name=String)++          Or alternatively::++              from sqlalchemy.sql import column+              stmt = text("SELECT * FROM table").columns(+                                    column('id', Integer),+                                    column('name', String)+                                )++          .. deprecated:: 0.9.0 the :meth:`.TextClause.columns` method+             supersedes the ``typemap`` argument to :func:`.text`.++        .. seealso::++            :ref:`sqlexpression_text` - in the Core tutorial++            :ref:`orm_tutorial_literal_sql` - in the ORM tutorial++        """+        stmt = TextClause(text, bind=bind)+        if bindparams:+            stmt = stmt.bindparams(*bindparams)+        if typemap:+            stmt = stmt.columns(**typemap)+        if autocommit is not None:+            util.warn_deprecated('autocommit on text() is deprecated.  '+                                 'Use .execution_options(autocommit=True)')+            stmt = stmt.execution_options(autocommit=autocommit)++        return stmt++    @_generative+    def bindparams(self, *binds, **names_to_values):+        """Establish the values and/or types of bound parameters within+        this :class:`.TextClause` construct.++        Given a text construct such as::++            from sqlalchemy import text+            stmt = text("SELECT id, name FROM user WHERE name=:name "+                        "AND timestamp=:timestamp")++        the :meth:`.TextClause.bindparams` method can be used to establish+        the initial value of ``:name`` and ``:timestamp``,+        using simple keyword arguments::++            stmt = stmt.bindparams(name='jack',+                        timestamp=datetime.datetime(2012, 10, 8, 15, 12, 5))++        Where above, new :class:`.BindParameter` objects+        will be generated with the names ``name`` and ``timestamp``, and+        values of ``jack`` and ``datetime.datetime(2012, 10, 8, 15, 12, 5)``,+        respectively.  The types will be+        inferred from the values given, in this case :class:`.String` and+        :class:`.DateTime`.++        When specific typing behavior is needed, the positional ``*binds``+        argument can be used in which to specify :func:`.bindparam` constructs+        directly.  These constructs must include at least the ``key``+        argument, then an optional value and type::++            from sqlalchemy import bindparam+            stmt = stmt.bindparams(+                            bindparam('name', value='jack', type_=String),+                            bindparam('timestamp', type_=DateTime)+                        )++        Above, we specified the type of :class:`.DateTime` for the+        ``timestamp`` bind, and the type of :class:`.String` for the ``name``+        bind.  In the case of ``name`` we also set the default value of+        ``"jack"``.++        Additional bound parameters can be supplied at statement execution+        time, e.g.::++            result = connection.execute(stmt,+                        timestamp=datetime.datetime(2012, 10, 8, 15, 12, 5))++        The :meth:`.TextClause.bindparams` method can be called repeatedly,+        where it will re-use existing :class:`.BindParameter` objects to add+        new information.  For example, we can call+        :meth:`.TextClause.bindparams` first with typing information, and a+        second time with value information, and it will be combined::++            stmt = text("SELECT id, name FROM user WHERE name=:name "+                        "AND timestamp=:timestamp")+            stmt = stmt.bindparams(+                bindparam('name', type_=String),+                bindparam('timestamp', type_=DateTime)+            )+            stmt = stmt.bindparams(+                name='jack',+                timestamp=datetime.datetime(2012, 10, 8, 15, 12, 5)+            )+++        .. versionadded:: 0.9.0 The :meth:`.TextClause.bindparams` method+           supersedes the argument ``bindparams`` passed to+           :func:`~.expression.text`.+++        """+        self._bindparams = new_params = self._bindparams.copy()++        for bind in binds:+            try:+                existing = new_params[bind.key]+            except KeyError:+                raise exc.ArgumentError(+                    "This text() construct doesn't define a "+                    "bound parameter named %r" % bind.key)+            else:+                new_params[existing.key] = bind++        for key, value in names_to_values.items():+            try:+                existing = new_params[key]+            except KeyError:+                raise exc.ArgumentError(+                    "This text() construct doesn't define a "+                    "bound parameter named %r" % key)+            else:+                new_params[key] = existing._with_value(value)++    @util.dependencies('sqlalchemy.sql.selectable')+    def columns(self, selectable, *cols, **types):+        """Turn this :class:`.TextClause` object into a :class:`.TextAsFrom`+        object that can be embedded into another statement.++        This function essentially bridges the gap between an entirely+        textual SELECT statement and the SQL expression language concept+        of a "selectable"::++            from sqlalchemy.sql import column, text++            stmt = text("SELECT id, name FROM some_table")+            stmt = stmt.columns(column('id'), column('name')).alias('st')++            stmt = select([mytable]).\+                    select_from(+                        mytable.join(stmt, mytable.c.name == stmt.c.name)+                    ).where(stmt.c.id > 5)++        Above, we pass a series of :func:`.column` elements to the+        :meth:`.TextClause.columns` method positionally.  These :func:`.column`+        elements now become first class elements upon the :attr:`.TextAsFrom.c`+        column collection, just like any other selectable.++        The column expressions we pass to :meth:`.TextClause.columns` may+        also be typed; when we do so, these :class:`.TypeEngine` objects become+        the effective return type of the column, so that SQLAlchemy's+        result-set-processing systems may be used on the return values.+        This is often needed for types such as date or boolean types, as well+        as for unicode processing on some dialect configurations::++            stmt = text("SELECT id, name, timestamp FROM some_table")+            stmt = stmt.columns(+                        column('id', Integer),+                        column('name', Unicode),+                        column('timestamp', DateTime)+                    )++            for id, name, timestamp in connection.execute(stmt):+                print(id, name, timestamp)++        As a shortcut to the above syntax, keyword arguments referring to+        types alone may be used, if only type conversion is needed::++            stmt = text("SELECT id, name, timestamp FROM some_table")+            stmt = stmt.columns(+                        id=Integer,+                        name=Unicode,+                        timestamp=DateTime+                    )++            for id, name, timestamp in connection.execute(stmt):+                print(id, name, timestamp)++        The positional form of :meth:`.TextClause.columns` also provides+        the unique feature of **positional column targeting**, which is+        particularly useful when using the ORM with complex textual queries.+        If we specify the columns from our model to :meth:`.TextClause.columns`,+        the result set will match to those columns positionally, meaning the+        name or origin of the column in the textual SQL doesn't matter::++            stmt = text("SELECT users.id, addresses.id, users.id, "+                 "users.name, addresses.email_address AS email "+                 "FROM users JOIN addresses ON users.id=addresses.user_id "+                 "WHERE users.id = 1").columns(+                    User.id,+                    Address.id,+                    Address.user_id,+                    User.name,+                    Address.email_address+                 )++            query = session.query(User).from_statement(stmt).options(+                contains_eager(User.addresses))++        .. versionadded:: 1.1 the :meth:`.TextClause.columns` method now+           offers positional column targeting in the result set when+           the column expressions are passed purely positionally.++        The :meth:`.TextClause.columns` method provides a direct+        route to calling :meth:`.FromClause.alias` as well as+        :meth:`.SelectBase.cte` against a textual SELECT statement::++            stmt = stmt.columns(id=Integer, name=String).cte('st')++            stmt = select([sometable]).where(sometable.c.id == stmt.c.id)++        .. versionadded:: 0.9.0 :func:`.text` can now be converted into a+           fully featured "selectable" construct using the+           :meth:`.TextClause.columns` method.  This method supersedes the+           ``typemap`` argument to :func:`.text`.+++        """++        positional_input_cols = [+            ColumnClause(col.key, types.pop(col.key))+            if col.key in types+            else col+            for col in cols+        ]+        keyed_input_cols = [+            ColumnClause(key, type_) for key, type_ in types.items()]++        return selectable.TextAsFrom(+            self,+            positional_input_cols + keyed_input_cols,+            positional=bool(positional_input_cols) and not keyed_input_cols)++    @property+    def type(self):+        return type_api.NULLTYPE++    @property+    def comparator(self):+        return self.type.comparator_factory(self)++    def self_group(self, against=None):+        if against is operators.in_op:+            return Grouping(self)+        else:+            return self++    def _copy_internals(self, clone=_clone, **kw):+        self._bindparams = dict((b.key, clone(b, **kw))+                                for b in self._bindparams.values())++    def get_children(self, **kwargs):+        return list(self._bindparams.values())++    def compare(self, other):+        return isinstance(other, TextClause) and other.text == self.text+++class Null(ColumnElement):+    """Represent the NULL keyword in a SQL statement.++    :class:`.Null` is accessed as a constant via the+    :func:`.null` function.++    """++    __visit_name__ = 'null'++    @util.memoized_property+    def type(self):+        return type_api.NULLTYPE++    @classmethod+    def _instance(cls):+        """Return a constant :class:`.Null` construct."""++        return Null()++    def compare(self, other):+        return isinstance(other, Null)+++class False_(ColumnElement):+    """Represent the ``false`` keyword, or equivalent, in a SQL statement.++    :class:`.False_` is accessed as a constant via the+    :func:`.false` function.++    """++    __visit_name__ = 'false'++    @util.memoized_property+    def type(self):+        return type_api.BOOLEANTYPE++    def _negate(self):+        return True_()++    @classmethod+    def _instance(cls):+        """Return a :class:`.False_` construct.++        E.g.::++            >>> from sqlalchemy import false+            >>> print select([t.c.x]).where(false())+            SELECT x FROM t WHERE false++        A backend which does not support true/false constants will render as+        an expression against 1 or 0::++            >>> print select([t.c.x]).where(false())+            SELECT x FROM t WHERE 0 = 1++        The :func:`.true` and :func:`.false` constants also feature+        "short circuit" operation within an :func:`.and_` or :func:`.or_`+        conjunction::++            >>> print select([t.c.x]).where(or_(t.c.x > 5, true()))+            SELECT x FROM t WHERE true++            >>> print select([t.c.x]).where(and_(t.c.x > 5, false()))+            SELECT x FROM t WHERE false++        .. versionchanged:: 0.9 :func:`.true` and :func:`.false` feature+           better integrated behavior within conjunctions and on dialects+           that don't support true/false constants.++        .. seealso::++            :func:`.true`++        """++        return False_()++    def compare(self, other):+        return isinstance(other, False_)+++class True_(ColumnElement):+    """Represent the ``true`` keyword, or equivalent, in a SQL statement.++    :class:`.True_` is accessed as a constant via the+    :func:`.true` function.++    """++    __visit_name__ = 'true'++    @util.memoized_property+    def type(self):+        return type_api.BOOLEANTYPE++    def _negate(self):+        return False_()++    @classmethod+    def _ifnone(cls, other):+        if other is None:+            return cls._instance()+        else:+            return other++    @classmethod+    def _instance(cls):+        """Return a constant :class:`.True_` construct.++        E.g.::++            >>> from sqlalchemy import true+            >>> print select([t.c.x]).where(true())+            SELECT x FROM t WHERE true++        A backend which does not support true/false constants will render as+        an expression against 1 or 0::++            >>> print select([t.c.x]).where(true())+            SELECT x FROM t WHERE 1 = 1++        The :func:`.true` and :func:`.false` constants also feature+        "short circuit" operation within an :func:`.and_` or :func:`.or_`+        conjunction::++            >>> print select([t.c.x]).where(or_(t.c.x > 5, true()))+            SELECT x FROM t WHERE true++            >>> print select([t.c.x]).where(and_(t.c.x > 5, false()))+            SELECT x FROM t WHERE false++        .. versionchanged:: 0.9 :func:`.true` and :func:`.false` feature+           better integrated behavior within conjunctions and on dialects+           that don't support true/false constants.++        .. seealso::++            :func:`.false`++        """++        return True_()++    def compare(self, other):+        return isinstance(other, True_)+++class ClauseList(ClauseElement):+    """Describe a list of clauses, separated by an operator.++    By default, is comma-separated, such as a column listing.++    """+    __visit_name__ = 'clauselist'++    def __init__(self, *clauses, **kwargs):+        self.operator = kwargs.pop('operator', operators.comma_op)+        self.group = kwargs.pop('group', True)+        self.group_contents = kwargs.pop('group_contents', True)+        text_converter = kwargs.pop(+            '_literal_as_text',+            _expression_literal_as_text)+        if self.group_contents:+            self.clauses = [+                text_converter(clause).self_group(against=self.operator)+                for clause in clauses]+        else:+            self.clauses = [+                text_converter(clause)+                for clause in clauses]++    def __iter__(self):+        return iter(self.clauses)++    def __len__(self):+        return len(self.clauses)++    @property+    def _select_iterable(self):+        return iter(self)++    def append(self, clause):+        if self.group_contents:+            self.clauses.append(_literal_as_text(clause).+                                self_group(against=self.operator))+        else:+            self.clauses.append(_literal_as_text(clause))++    def _copy_internals(self, clone=_clone, **kw):+        self.clauses = [clone(clause, **kw) for clause in self.clauses]++    def get_children(self, **kwargs):+        return self.clauses++    @property+    def _from_objects(self):+        return list(itertools.chain(*[c._from_objects for c in self.clauses]))++    def self_group(self, against=None):+        if self.group and operators.is_precedent(self.operator, against):+            return Grouping(self)+        else:+            return self++    def compare(self, other, **kw):+        """Compare this :class:`.ClauseList` to the given :class:`.ClauseList`,+        including a comparison of all the clause items.++        """+        if not isinstance(other, ClauseList) and len(self.clauses) == 1:+            return self.clauses[0].compare(other, **kw)+        elif isinstance(other, ClauseList) and \+                len(self.clauses) == len(other.clauses) and \+                self.operator is other.operator:++            if self.operator in (operators.and_, operators.or_):+                completed = set()+                for clause in self.clauses:+                    for other_clause in set(other.clauses).difference(completed):+                        if clause.compare(other_clause, **kw):+                            completed.add(other_clause)+                            break+                return len(completed) == len(other.clauses)+            else:+                for i in range(0, len(self.clauses)):+                    if not self.clauses[i].compare(other.clauses[i], **kw):+                        return False+                else:+                    return True+        else:+            return False+++class BooleanClauseList(ClauseList, ColumnElement):+    __visit_name__ = 'clauselist'++    def __init__(self, *arg, **kw):+        raise NotImplementedError(+            "BooleanClauseList has a private constructor")++    @classmethod+    def _construct(cls, operator, continue_on, skip_on, *clauses, **kw):+        convert_clauses = []++        clauses = [+            _expression_literal_as_text(clause)+            for clause in+            util.coerce_generator_arg(clauses)+        ]+        for clause in clauses:++            if isinstance(clause, continue_on):+                continue+            elif isinstance(clause, skip_on):+                return clause.self_group(against=operators._asbool)++            convert_clauses.append(clause)++        if len(convert_clauses) == 1:+            return convert_clauses[0].self_group(against=operators._asbool)+        elif not convert_clauses and clauses:+            return clauses[0].self_group(against=operators._asbool)++        convert_clauses = [c.self_group(against=operator)+                           for c in convert_clauses]++        self = cls.__new__(cls)+        self.clauses = convert_clauses+        self.group = True+        self.operator = operator+        self.group_contents = True+        self.type = type_api.BOOLEANTYPE+        return self++    @classmethod+    def and_(cls, *clauses):+        """Produce a conjunction of expressions joined by ``AND``.++        E.g.::++            from sqlalchemy import and_++            stmt = select([users_table]).where(+                            and_(+                                users_table.c.name == 'wendy',+                                users_table.c.enrolled == True+                            )+                        )++        The :func:`.and_` conjunction is also available using the+        Python ``&`` operator (though note that compound expressions+        need to be parenthesized in order to function with Python+        operator precedence behavior)::++            stmt = select([users_table]).where(+                            (users_table.c.name == 'wendy') &+                            (users_table.c.enrolled == True)+                        )++        The :func:`.and_` operation is also implicit in some cases;+        the :meth:`.Select.where` method for example can be invoked multiple+        times against a statement, which will have the effect of each+        clause being combined using :func:`.and_`::++            stmt = select([users_table]).\+                        where(users_table.c.name == 'wendy').\+                        where(users_table.c.enrolled == True)++        .. seealso::++            :func:`.or_`++        """+        return cls._construct(operators.and_, True_, False_, *clauses)++    @classmethod+    def or_(cls, *clauses):+        """Produce a conjunction of expressions joined by ``OR``.++        E.g.::++            from sqlalchemy import or_++            stmt = select([users_table]).where(+                            or_(+                                users_table.c.name == 'wendy',+                                users_table.c.name == 'jack'+                            )+                        )++        The :func:`.or_` conjunction is also available using the+        Python ``|`` operator (though note that compound expressions+        need to be parenthesized in order to function with Python+        operator precedence behavior)::++            stmt = select([users_table]).where(+                            (users_table.c.name == 'wendy') |+                            (users_table.c.name == 'jack')+                        )++        .. seealso::++            :func:`.and_`++        """+        return cls._construct(operators.or_, False_, True_, *clauses)++    @property+    def _select_iterable(self):+        return (self, )++    def self_group(self, against=None):+        if not self.clauses:+            return self+        else:+            return super(BooleanClauseList, self).self_group(against=against)++    def _negate(self):+        return ClauseList._negate(self)+++and_ = BooleanClauseList.and_+or_ = BooleanClauseList.or_+++class Tuple(ClauseList, ColumnElement):+    """Represent a SQL tuple."""++    def __init__(self, *clauses, **kw):+        """Return a :class:`.Tuple`.++        Main usage is to produce a composite IN construct::++            from sqlalchemy import tuple_++            tuple_(table.c.col1, table.c.col2).in_(+                [(1, 2), (5, 12), (10, 19)]+            )++        .. warning::++            The composite IN construct is not supported by all backends,+            and is currently known to work on PostgreSQL and MySQL,+            but not SQLite.   Unsupported backends will raise+            a subclass of :class:`~sqlalchemy.exc.DBAPIError` when such+            an expression is invoked.++        """++        clauses = [_literal_as_binds(c) for c in clauses]+        self._type_tuple = [arg.type for arg in clauses]+        self.type = kw.pop('type_', self._type_tuple[0]+                           if self._type_tuple else type_api.NULLTYPE)++        super(Tuple, self).__init__(*clauses, **kw)++    @property+    def _select_iterable(self):+        return (self, )++    def _bind_param(self, operator, obj, type_=None):+        return Tuple(*[+            BindParameter(None, o, _compared_to_operator=operator,+                          _compared_to_type=compared_to_type, unique=True,+                          type_=type_)+            for o, compared_to_type in zip(obj, self._type_tuple)+        ]).self_group()+++class Case(ColumnElement):+    """Represent a ``CASE`` expression.++    :class:`.Case` is produced using the :func:`.case` factory function,+    as in::++        from sqlalchemy import case++        stmt = select([users_table]).\+                    where(+                        case(+                            [+                                (users_table.c.name == 'wendy', 'W'),+                                (users_table.c.name == 'jack', 'J')+                            ],+                            else_='E'+                        )+                    )++    Details on :class:`.Case` usage is at :func:`.case`.++    .. seealso::++        :func:`.case`++    """++    __visit_name__ = 'case'++    def __init__(self, whens, value=None, else_=None):+        r"""Produce a ``CASE`` expression.++        The ``CASE`` construct in SQL is a conditional object that+        acts somewhat analogously to an "if/then" construct in other+        languages.  It returns an instance of :class:`.Case`.++        :func:`.case` in its usual form is passed a list of "when"+        constructs, that is, a list of conditions and results as tuples::++            from sqlalchemy import case++            stmt = select([users_table]).\+                        where(+                            case(+                                [+                                    (users_table.c.name == 'wendy', 'W'),+                                    (users_table.c.name == 'jack', 'J')+                                ],+                                else_='E'+                            )+                        )++        The above statement will produce SQL resembling::++            SELECT id, name FROM user+            WHERE CASE+                WHEN (name = :name_1) THEN :param_1+                WHEN (name = :name_2) THEN :param_2+                ELSE :param_3+            END++        When simple equality expressions of several values against a single+        parent column are needed, :func:`.case` also has a "shorthand" format+        used via the+        :paramref:`.case.value` parameter, which is passed a column+        expression to be compared.  In this form, the :paramref:`.case.whens`+        parameter is passed as a dictionary containing expressions to be+        compared against keyed to result expressions.  The statement below is+        equivalent to the preceding statement::++            stmt = select([users_table]).\+                        where(+                            case(+                                {"wendy": "W", "jack": "J"},+                                value=users_table.c.name,+                                else_='E'+                            )+                        )++        The values which are accepted as result values in+        :paramref:`.case.whens` as well as with :paramref:`.case.else_` are+        coerced from Python literals into :func:`.bindparam` constructs.+        SQL expressions, e.g. :class:`.ColumnElement` constructs, are accepted+        as well.  To coerce a literal string expression into a constant+        expression rendered inline, use the :func:`.literal_column` construct,+        as in::++            from sqlalchemy import case, literal_column++            case(+                [+                    (+                        orderline.c.qty > 100,+                        literal_column("'greaterthan100'")+                    ),+                    (+                        orderline.c.qty > 10,+                        literal_column("'greaterthan10'")+                    )+                ],+                else_=literal_column("'lessthan10'")+            )++        The above will render the given constants without using bound+        parameters for the result values (but still for the comparison+        values), as in::++            CASE+                WHEN (orderline.qty > :qty_1) THEN 'greaterthan100'+                WHEN (orderline.qty > :qty_2) THEN 'greaterthan10'+                ELSE 'lessthan10'+            END++        :param whens: The criteria to be compared against,+         :paramref:`.case.whens` accepts two different forms, based on+         whether or not :paramref:`.case.value` is used.++         In the first form, it accepts a list of 2-tuples; each 2-tuple+         consists of ``(<sql expression>, <value>)``, where the SQL+         expression is a boolean expression and "value" is a resulting value,+         e.g.::++            case([+                (users_table.c.name == 'wendy', 'W'),+                (users_table.c.name == 'jack', 'J')+            ])++         In the second form, it accepts a Python dictionary of comparison+         values mapped to a resulting value; this form requires+         :paramref:`.case.value` to be present, and values will be compared+         using the ``==`` operator, e.g.::++            case(+                {"wendy": "W", "jack": "J"},+                value=users_table.c.name+            )++        :param value: An optional SQL expression which will be used as a+          fixed "comparison point" for candidate values within a dictionary+          passed to :paramref:`.case.whens`.++        :param else\_: An optional SQL expression which will be the evaluated+          result of the ``CASE`` construct if all expressions within+          :paramref:`.case.whens` evaluate to false.  When omitted, most+          databases will produce a result of NULL if none of the "when"+          expressions evaluate to true.+++        """++        try:+            whens = util.dictlike_iteritems(whens)+        except TypeError:+            pass++        if value is not None:+            whenlist = [+                (_literal_as_binds(c).self_group(),+                 _literal_as_binds(r)) for (c, r) in whens+            ]+        else:+            whenlist = [+                (_no_literals(c).self_group(),+                 _literal_as_binds(r)) for (c, r) in whens+            ]++        if whenlist:+            type_ = list(whenlist[-1])[-1].type+        else:+            type_ = None++        if value is None:+            self.value = None+        else:+            self.value = _literal_as_binds(value)++        self.type = type_+        self.whens = whenlist+        if else_ is not None:+            self.else_ = _literal_as_binds(else_)+        else:+            self.else_ = None++    def _copy_internals(self, clone=_clone, **kw):+        if self.value is not None:+            self.value = clone(self.value, **kw)+        self.whens = [(clone(x, **kw), clone(y, **kw))+                      for x, y in self.whens]+        if self.else_ is not None:+            self.else_ = clone(self.else_, **kw)++    def get_children(self, **kwargs):+        if self.value is not None:+            yield self.value+        for x, y in self.whens:+            yield x+            yield y+        if self.else_ is not None:+            yield self.else_++    @property+    def _from_objects(self):+        return list(itertools.chain(*[x._from_objects for x in+                                      self.get_children()]))+++def literal_column(text, type_=None):+    r"""Produce a :class:`.ColumnClause` object that has the+    :paramref:`.column.is_literal` flag set to True.++    :func:`.literal_column` is similar to :func:`.column`, except that+    it is more often used as a "standalone" column expression that renders+    exactly as stated; while :func:`.column` stores a string name that+    will be assumed to be part of a table and may be quoted as such,+    :func:`.literal_column` can be that, or any other arbitrary column-oriented+    expression.++    :param text: the text of the expression; can be any SQL expression.+      Quoting rules will not be applied. To specify a column-name expression+      which should be subject to quoting rules, use the :func:`column`+      function.++    :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine`+      object which will+      provide result-set translation and additional expression semantics for+      this column. If left as None the type will be NullType.++    .. seealso::++        :func:`.column`++        :func:`.text`++        :ref:`sqlexpression_literal_column`++    """+    return ColumnClause(text, type_=type_, is_literal=True)+++class Cast(ColumnElement):+    """Represent a ``CAST`` expression.++    :class:`.Cast` is produced using the :func:`.cast` factory function,+    as in::++        from sqlalchemy import cast, Numeric++        stmt = select([+                    cast(product_table.c.unit_price, Numeric(10, 4))+                ])++    Details on :class:`.Cast` usage is at :func:`.cast`.++    .. seealso::++        :func:`.cast`++    """++    __visit_name__ = 'cast'++    def __init__(self, expression, type_):+        """Produce a ``CAST`` expression.++        :func:`.cast` returns an instance of :class:`.Cast`.++        E.g.::++            from sqlalchemy import cast, Numeric++            stmt = select([+                        cast(product_table.c.unit_price, Numeric(10, 4))+                    ])++        The above statement will produce SQL resembling::++            SELECT CAST(unit_price AS NUMERIC(10, 4)) FROM product++        The :func:`.cast` function performs two distinct functions when+        used.  The first is that it renders the ``CAST`` expression within+        the resulting SQL string.  The second is that it associates the given+        type (e.g. :class:`.TypeEngine` class or instance) with the column+        expression on the Python side, which means the expression will take+        on the expression operator behavior associated with that type,+        as well as the bound-value handling and result-row-handling behavior+        of the type.++        .. versionchanged:: 0.9.0 :func:`.cast` now applies the given type+           to the expression such that it takes effect on the bound-value,+           e.g. the Python-to-database direction, in addition to the+           result handling, e.g. database-to-Python, direction.++        An alternative to :func:`.cast` is the :func:`.type_coerce` function.+        This function performs the second task of associating an expression+        with a specific type, but does not render the ``CAST`` expression+        in SQL.++        :param expression: A SQL expression, such as a :class:`.ColumnElement`+         expression or a Python string which will be coerced into a bound+         literal value.++        :param type_: A :class:`.TypeEngine` class or instance indicating+         the type to which the ``CAST`` should apply.++        .. seealso::++            :func:`.type_coerce` - Python-side type coercion without emitting+            CAST.++        """+        self.type = type_api.to_instance(type_)+        self.clause = _literal_as_binds(expression, type_=self.type)+        self.typeclause = TypeClause(self.type)++    def _copy_internals(self, clone=_clone, **kw):+        self.clause = clone(self.clause, **kw)+        self.typeclause = clone(self.typeclause, **kw)++    def get_children(self, **kwargs):+        return self.clause, self.typeclause++    @property+    def _from_objects(self):+        return self.clause._from_objects+++class TypeCoerce(ColumnElement):+    """Represent a Python-side type-coercion wrapper.++    :class:`.TypeCoerce` supplies the :func:`.expression.type_coerce`+    function; see that function for usage details.++    .. versionchanged:: 1.1 The :func:`.type_coerce` function now produces+       a persistent :class:`.TypeCoerce` wrapper object rather than+       translating the given object in place.++    .. seealso::++        :func:`.expression.type_coerce`++    """++    __visit_name__ = 'type_coerce'++    def __init__(self, expression, type_):+        """Associate a SQL expression with a particular type, without rendering+        ``CAST``.++        E.g.::++            from sqlalchemy import type_coerce++            stmt = select([+                type_coerce(log_table.date_string, StringDateTime())+            ])++        The above construct will produce a :class:`.TypeCoerce` object, which+        renders SQL that labels the expression, but otherwise does not+        modify its value on the SQL side::++            SELECT date_string AS anon_1 FROM log++        When result rows are fetched, the ``StringDateTime`` type+        will be applied to result rows on behalf of the ``date_string`` column.+        The rationale for the "anon_1" label is so that the type-coerced+        column remains separate in the list of result columns vs. other+        type-coerced or direct values of the target column.  In order to+        provide a named label for the expression, use+        :meth:`.ColumnElement.label`::++            stmt = select([+                type_coerce(+                    log_table.date_string, StringDateTime()).label('date')+            ])+++        A type that features bound-value handling will also have that behavior+        take effect when literal values or :func:`.bindparam` constructs are+        passed to :func:`.type_coerce` as targets.+        For example, if a type implements the+        :meth:`.TypeEngine.bind_expression`+        method or :meth:`.TypeEngine.bind_processor` method or equivalent,+        these functions will take effect at statement compilation/execution+        time when a literal value is passed, as in::++            # bound-value handling of MyStringType will be applied to the+            # literal value "some string"+            stmt = select([type_coerce("some string", MyStringType)])++        :func:`.type_coerce` is similar to the :func:`.cast` function,+        except that it does not render the ``CAST`` expression in the resulting+        statement.++        :param expression: A SQL expression, such as a :class:`.ColumnElement`+         expression or a Python string which will be coerced into a bound+         literal value.++        :param type_: A :class:`.TypeEngine` class or instance indicating+         the type to which the expression is coerced.++        .. seealso::++            :func:`.cast`++        """+        self.type = type_api.to_instance(type_)+        self.clause = _literal_as_binds(expression, type_=self.type)++    def _copy_internals(self, clone=_clone, **kw):+        self.clause = clone(self.clause, **kw)+        self.__dict__.pop('typed_expression', None)++    def get_children(self, **kwargs):+        return self.clause,++    @property+    def _from_objects(self):+        return self.clause._from_objects++    @util.memoized_property+    def typed_expression(self):+        if isinstance(self.clause, BindParameter):+            bp = self.clause._clone()+            bp.type = self.type+            return bp+        else:+            return self.clause+++class Extract(ColumnElement):+    """Represent a SQL EXTRACT clause, ``extract(field FROM expr)``."""++    __visit_name__ = 'extract'++    def __init__(self, field, expr, **kwargs):+        """Return a :class:`.Extract` construct.++        This is typically available as :func:`.extract`+        as well as ``func.extract`` from the+        :data:`.func` namespace.++        """+        self.type = type_api.INTEGERTYPE+        self.field = field+        self.expr = _literal_as_binds(expr, None)++    def _copy_internals(self, clone=_clone, **kw):+        self.expr = clone(self.expr, **kw)++    def get_children(self, **kwargs):+        return self.expr,++    @property+    def _from_objects(self):+        return self.expr._from_objects+++class _label_reference(ColumnElement):+    """Wrap a column expression as it appears in a 'reference' context.++    This expression is any that includes an _order_by_label_element,+    which is a Label, or a DESC / ASC construct wrapping a Label.++    The production of _label_reference() should occur when an expression+    is added to this context; this includes the ORDER BY or GROUP BY of a+    SELECT statement, as well as a few other places, such as the ORDER BY+    within an OVER clause.++    """+    __visit_name__ = 'label_reference'++    def __init__(self, element):+        self.element = element++    def _copy_internals(self, clone=_clone, **kw):+        self.element = clone(self.element, **kw)++    @property+    def _from_objects(self):+        return ()+++class _textual_label_reference(ColumnElement):+    __visit_name__ = 'textual_label_reference'++    def __init__(self, element):+        self.element = element++    @util.memoized_property+    def _text_clause(self):+        return TextClause._create_text(self.element)+++class UnaryExpression(ColumnElement):+    """Define a 'unary' expression.++    A unary expression has a single column expression+    and an operator.  The operator can be placed on the left+    (where it is called the 'operator') or right (where it is called the+    'modifier') of the column expression.++    :class:`.UnaryExpression` is the basis for several unary operators+    including those used by :func:`.desc`, :func:`.asc`, :func:`.distinct`,+    :func:`.nullsfirst` and :func:`.nullslast`.++    """+    __visit_name__ = 'unary'++    def __init__(self, element, operator=None, modifier=None,+                 type_=None, negate=None, wraps_column_expression=False):+        self.operator = operator+        self.modifier = modifier+        self.element = element.self_group(+            against=self.operator or self.modifier)+        self.type = type_api.to_instance(type_)+        self.negate = negate+        self.wraps_column_expression = wraps_column_expression++    @classmethod+    def _create_nullsfirst(cls, column):+        """Produce the ``NULLS FIRST`` modifier for an ``ORDER BY`` expression.++        :func:`.nullsfirst` is intended to modify the expression produced+        by :func:`.asc` or :func:`.desc`, and indicates how NULL values+        should be handled when they are encountered during ordering::+++            from sqlalchemy import desc, nullsfirst++            stmt = select([users_table]).\+                        order_by(nullsfirst(desc(users_table.c.name)))++        The SQL expression from the above would resemble::++            SELECT id, name FROM user ORDER BY name DESC NULLS FIRST++        Like :func:`.asc` and :func:`.desc`, :func:`.nullsfirst` is typically+        invoked from the column expression itself using+        :meth:`.ColumnElement.nullsfirst`, rather than as its standalone+        function version, as in::++            stmt = (select([users_table]).+                    order_by(users_table.c.name.desc().nullsfirst())+                    )++        .. seealso::++            :func:`.asc`++            :func:`.desc`++            :func:`.nullslast`++            :meth:`.Select.order_by`++        """+        return UnaryExpression(+            _literal_as_label_reference(column),+            modifier=operators.nullsfirst_op,+            wraps_column_expression=False)++    @classmethod+    def _create_nullslast(cls, column):+        """Produce the ``NULLS LAST`` modifier for an ``ORDER BY`` expression.++        :func:`.nullslast` is intended to modify the expression produced+        by :func:`.asc` or :func:`.desc`, and indicates how NULL values+        should be handled when they are encountered during ordering::+++            from sqlalchemy import desc, nullslast++            stmt = select([users_table]).\+                        order_by(nullslast(desc(users_table.c.name)))++        The SQL expression from the above would resemble::++            SELECT id, name FROM user ORDER BY name DESC NULLS LAST++        Like :func:`.asc` and :func:`.desc`, :func:`.nullslast` is typically+        invoked from the column expression itself using+        :meth:`.ColumnElement.nullslast`, rather than as its standalone+        function version, as in::++            stmt = select([users_table]).\+                        order_by(users_table.c.name.desc().nullslast())++        .. seealso::++            :func:`.asc`++            :func:`.desc`++            :func:`.nullsfirst`++            :meth:`.Select.order_by`++        """+        return UnaryExpression(+            _literal_as_label_reference(column),+            modifier=operators.nullslast_op,+            wraps_column_expression=False)++    @classmethod+    def _create_desc(cls, column):+        """Produce a descending ``ORDER BY`` clause element.++        e.g.::++            from sqlalchemy import desc++            stmt = select([users_table]).order_by(desc(users_table.c.name))++        will produce SQL as::++            SELECT id, name FROM user ORDER BY name DESC++        The :func:`.desc` function is a standalone version of the+        :meth:`.ColumnElement.desc` method available on all SQL expressions,+        e.g.::+++            stmt = select([users_table]).order_by(users_table.c.name.desc())++        :param column: A :class:`.ColumnElement` (e.g. scalar SQL expression)+         with which to apply the :func:`.desc` operation.++        .. seealso::++            :func:`.asc`++            :func:`.nullsfirst`++            :func:`.nullslast`++            :meth:`.Select.order_by`++        """+        return UnaryExpression(+            _literal_as_label_reference(column),+            modifier=operators.desc_op,+            wraps_column_expression=False)++    @classmethod+    def _create_asc(cls, column):+        """Produce an ascending ``ORDER BY`` clause element.++        e.g.::++            from sqlalchemy import asc+            stmt = select([users_table]).order_by(asc(users_table.c.name))++        will produce SQL as::++            SELECT id, name FROM user ORDER BY name ASC++        The :func:`.asc` function is a standalone version of the+        :meth:`.ColumnElement.asc` method available on all SQL expressions,+        e.g.::+++            stmt = select([users_table]).order_by(users_table.c.name.asc())++        :param column: A :class:`.ColumnElement` (e.g. scalar SQL expression)+         with which to apply the :func:`.asc` operation.++        .. seealso::++            :func:`.desc`++            :func:`.nullsfirst`++            :func:`.nullslast`++            :meth:`.Select.order_by`++        """+        return UnaryExpression(+            _literal_as_label_reference(column),+            modifier=operators.asc_op,+            wraps_column_expression=False)++    @classmethod+    def _create_distinct(cls, expr):+        """Produce an column-expression-level unary ``DISTINCT`` clause.++        This applies the ``DISTINCT`` keyword to an individual column+        expression, and is typically contained within an aggregate function,+        as in::++            from sqlalchemy import distinct, func+            stmt = select([func.count(distinct(users_table.c.name))])++        The above would produce an expression resembling::++            SELECT COUNT(DISTINCT name) FROM user++        The :func:`.distinct` function is also available as a column-level+        method, e.g. :meth:`.ColumnElement.distinct`, as in::++            stmt = select([func.count(users_table.c.name.distinct())])++        The :func:`.distinct` operator is different from the+        :meth:`.Select.distinct` method of :class:`.Select`,+        which produces a ``SELECT`` statement+        with ``DISTINCT`` applied to the result set as a whole,+        e.g. a ``SELECT DISTINCT`` expression.  See that method for further+        information.++        .. seealso::++            :meth:`.ColumnElement.distinct`++            :meth:`.Select.distinct`++            :data:`.func`++        """+        expr = _literal_as_binds(expr)+        return UnaryExpression(+            expr, operator=operators.distinct_op,+            type_=expr.type, wraps_column_expression=False)++    @property+    def _order_by_label_element(self):+        if self.modifier in (operators.desc_op, operators.asc_op):+            return self.element._order_by_label_element+        else:+            return None++    @property+    def _from_objects(self):+        return self.element._from_objects++    def _copy_internals(self, clone=_clone, **kw):+        self.element = clone(self.element, **kw)++    def get_children(self, **kwargs):+        return self.element,++    def compare(self, other, **kw):+        """Compare this :class:`UnaryExpression` against the given+        :class:`.ClauseElement`."""++        return (+            isinstance(other, UnaryExpression) and+            self.operator == other.operator and+            self.modifier == other.modifier and+            self.element.compare(other.element, **kw)+        )++    def _negate(self):+        if self.negate is not None:+            return UnaryExpression(+                self.element,+                operator=self.negate,+                negate=self.operator,+                modifier=self.modifier,+                type_=self.type,+                wraps_column_expression=self.wraps_column_expression)+        elif self.type._type_affinity is type_api.BOOLEANTYPE._type_affinity:+            return UnaryExpression(+                self.self_group(against=operators.inv),+                operator=operators.inv,+                type_=type_api.BOOLEANTYPE,+                wraps_column_expression=self.wraps_column_expression,+                negate=None)+        else:+            return ClauseElement._negate(self)++    def self_group(self, against=None):+        if self.operator and operators.is_precedent(self.operator, against):+            return Grouping(self)+        else:+            return self+++class CollectionAggregate(UnaryExpression):+    """Forms the basis for right-hand collection operator modifiers+    ANY and ALL.++    The ANY and ALL keywords are available in different ways on different+    backends.  On PostgreSQL, they only work for an ARRAY type.  On+    MySQL, they only work for subqueries.++    """+    @classmethod+    def _create_any(cls, expr):+        """Produce an ANY expression.++        This may apply to an array type for some dialects (e.g. postgresql),+        or to a subquery for others (e.g. mysql).  e.g.::++            # postgresql '5 = ANY (somearray)'+            expr = 5 == any_(mytable.c.somearray)++            # mysql '5 = ANY (SELECT value FROM table)'+            expr = 5 == any_(select([table.c.value]))++        .. versionadded:: 1.1++        .. seealso::++            :func:`.expression.all_`++        """++        expr = _literal_as_binds(expr)++        if expr.is_selectable and hasattr(expr, 'as_scalar'):+            expr = expr.as_scalar()+        expr = expr.self_group()+        return CollectionAggregate(+            expr, operator=operators.any_op,+            type_=type_api.NULLTYPE, wraps_column_expression=False)++    @classmethod+    def _create_all(cls, expr):+        """Produce an ALL expression.++        This may apply to an array type for some dialects (e.g. postgresql),+        or to a subquery for others (e.g. mysql).  e.g.::++            # postgresql '5 = ALL (somearray)'+            expr = 5 == all_(mytable.c.somearray)++            # mysql '5 = ALL (SELECT value FROM table)'+            expr = 5 == all_(select([table.c.value]))++        .. versionadded:: 1.1++        .. seealso::++            :func:`.expression.any_`++        """++        expr = _literal_as_binds(expr)+        if expr.is_selectable and hasattr(expr, 'as_scalar'):+            expr = expr.as_scalar()+        expr = expr.self_group()+        return CollectionAggregate(+            expr, operator=operators.all_op,+            type_=type_api.NULLTYPE, wraps_column_expression=False)++    # operate and reverse_operate are hardwired to+    # dispatch onto the type comparator directly, so that we can+    # ensure "reversed" behavior.+    def operate(self, op, *other, **kwargs):+        if not operators.is_comparison(op):+            raise exc.ArgumentError(+                "Only comparison operators may be used with ANY/ALL")+        kwargs['reverse'] = True+        return self.comparator.operate(operators.mirror(op), *other, **kwargs)++    def reverse_operate(self, op, other, **kwargs):+        # comparison operators should never call reverse_operate+        assert not operators.is_comparison(op)+        raise exc.ArgumentError(+            "Only comparison operators may be used with ANY/ALL")+++class AsBoolean(UnaryExpression):++    def __init__(self, element, operator, negate):+        self.element = element+        self.type = type_api.BOOLEANTYPE+        self.operator = operator+        self.negate = negate+        self.modifier = None+        self.wraps_column_expression = True++    def self_group(self, against=None):+        return self++    def _negate(self):+        # TODO: this assumes the element is the True_() or False_()+        # object, but this assumption isn't enforced and+        # ColumnElement._negate() can send any number of expressions here+        return self.element._negate()+++class BinaryExpression(ColumnElement):+    """Represent an expression that is ``LEFT <operator> RIGHT``.++    A :class:`.BinaryExpression` is generated automatically+    whenever two column expressions are used in a Python binary expression::++        >>> from sqlalchemy.sql import column+        >>> column('a') + column('b')+        <sqlalchemy.sql.expression.BinaryExpression object at 0x101029dd0>+        >>> print column('a') + column('b')+        a + b++    """++    __visit_name__ = 'binary'++    def __init__(self, left, right, operator, type_=None,+                 negate=None, modifiers=None):+        # allow compatibility with libraries that+        # refer to BinaryExpression directly and pass strings+        if isinstance(operator, util.string_types):+            operator = operators.custom_op(operator)+        self._orig = (left, right)+        self.left = left.self_group(against=operator)+        self.right = right.self_group(against=operator)+        self.operator = operator+        self.type = type_api.to_instance(type_)+        self.negate = negate++        if modifiers is None:+            self.modifiers = {}+        else:+            self.modifiers = modifiers++    def __bool__(self):+        if self.operator in (operator.eq, operator.ne):+            return self.operator(hash(self._orig[0]), hash(self._orig[1]))+        else:+            raise TypeError("Boolean value of this clause is not defined")++    __nonzero__ = __bool__++    @property+    def is_comparison(self):+        return operators.is_comparison(self.operator)++    @property+    def _from_objects(self):+        return self.left._from_objects + self.right._from_objects++    def _copy_internals(self, clone=_clone, **kw):+        self.left = clone(self.left, **kw)+        self.right = clone(self.right, **kw)++    def get_children(self, **kwargs):+        return self.left, self.right++    def compare(self, other, **kw):+        """Compare this :class:`BinaryExpression` against the+        given :class:`BinaryExpression`."""++        return (+            isinstance(other, BinaryExpression) and+            self.operator == other.operator and+            (+                self.left.compare(other.left, **kw) and+                self.right.compare(other.right, **kw) or+                (+                    operators.is_commutative(self.operator) and+                    self.left.compare(other.right, **kw) and+                    self.right.compare(other.left, **kw)+                )+            )+        )++    def self_group(self, against=None):+        if operators.is_precedent(self.operator, against):+            return Grouping(self)+        else:+            return self++    def _negate(self):+        if self.negate is not None:+            return BinaryExpression(+                self.left,+                self.right,+                self.negate,+                negate=self.operator,+                type_=self.type,+                modifiers=self.modifiers)+        else:+            return super(BinaryExpression, self)._negate()+++class Slice(ColumnElement):+    """Represent SQL for a Python array-slice object.++    This is not a specific SQL construct at this level, but+    may be interpreted by specific dialects, e.g. PostgreSQL.++    """+    __visit_name__ = 'slice'++    def __init__(self, start, stop, step):+        self.start = start+        self.stop = stop+        self.step = step+        self.type = type_api.NULLTYPE++    def self_group(self, against=None):+        assert against is operator.getitem+        return self+++class IndexExpression(BinaryExpression):+    """Represent the class of expressions that are like an "index" operation.+    """+    pass+++class Grouping(ColumnElement):+    """Represent a grouping within a column expression"""++    __visit_name__ = 'grouping'++    def __init__(self, element):+        self.element = element+        self.type = getattr(element, 'type', type_api.NULLTYPE)++    def self_group(self, against=None):+        return self++    @property+    def _key_label(self):+        return self._label++    @property+    def _label(self):+        return getattr(self.element, '_label', None) or self.anon_label++    def _copy_internals(self, clone=_clone, **kw):+        self.element = clone(self.element, **kw)++    def get_children(self, **kwargs):+        return self.element,++    @property+    def _from_objects(self):+        return self.element._from_objects++    def __getattr__(self, attr):+        return getattr(self.element, attr)++    def __getstate__(self):+        return {'element': self.element, 'type': self.type}++    def __setstate__(self, state):+        self.element = state['element']+        self.type = state['type']++    def compare(self, other, **kw):+        return isinstance(other, Grouping) and \+            self.element.compare(other.element)+++RANGE_UNBOUNDED = util.symbol("RANGE_UNBOUNDED")+RANGE_CURRENT = util.symbol("RANGE_CURRENT")+++class Over(ColumnElement):+    """Represent an OVER clause.++    This is a special operator against a so-called+    "window" function, as well as any aggregate function,+    which produces results relative to the result set+    itself.  It's supported only by certain database+    backends.++    """+    __visit_name__ = 'over'++    order_by = None+    partition_by = None++    def __init__(+            self, element, partition_by=None,+            order_by=None, range_=None, rows=None):+        """Produce an :class:`.Over` object against a function.++        Used against aggregate or so-called "window" functions,+        for database backends that support window functions.++        :func:`~.expression.over` is usually called using+        the :meth:`.FunctionElement.over` method, e.g.::++            func.row_number().over(order_by=mytable.c.some_column)++        Would produce::++            ROW_NUMBER() OVER(ORDER BY some_column)++        Ranges are also possible using the :paramref:`.expression.over.range_`+        and :paramref:`.expression.over.rows` parameters.  These+        mutually-exclusive parameters each accept a 2-tuple, which contains+        a combination of integers and None::++            func.row_number().over(order_by=my_table.c.some_column, range_=(None, 0))++        The above would produce::++            ROW_NUMBER() OVER(ORDER BY some_column RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)++        A value of None indicates "unbounded", a+        value of zero indicates "current row", and negative / positive+        integers indicate "preceding" and "following":++        * RANGE BETWEEN 5 PRECEDING AND 10 FOLLOWING::++            func.row_number().over(order_by='x', range_=(-5, 10))++        * ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW::++            func.row_number().over(order_by='x', rows=(None, 0))++        * RANGE BETWEEN 2 PRECEDING AND UNBOUNDED FOLLOWING::++            func.row_number().over(order_by='x', range_=(-2, None))++        * RANGE BETWEEN 1 FOLLOWING AND 3 FOLLOWING::++            func.row_number().over(order_by='x', range_=(1, 3))++        .. versionadded:: 1.1 support for RANGE / ROWS within a window+++        :param element: a :class:`.FunctionElement`, :class:`.WithinGroup`,+         or other compatible construct.+        :param partition_by: a column element or string, or a list+         of such, that will be used as the PARTITION BY clause+         of the OVER construct.+        :param order_by: a column element or string, or a list+         of such, that will be used as the ORDER BY clause+         of the OVER construct.+        :param range_: optional range clause for the window.  This is a+         tuple value which can contain integer values or None, and will+         render a RANGE BETWEEN PRECEDING / FOLLOWING clause++         .. versionadded:: 1.1++        :param rows: optional rows clause for the window.  This is a tuple+         value which can contain integer values or None, and will render+         a ROWS BETWEEN PRECEDING / FOLLOWING clause.++         .. versionadded:: 1.1++        This function is also available from the :data:`~.expression.func`+        construct itself via the :meth:`.FunctionElement.over` method.++        .. seealso::++            :data:`.expression.func`++            :func:`.expression.within_group`++        """+        self.element = element+        if order_by is not None:+            self.order_by = ClauseList(+                *util.to_list(order_by),+                _literal_as_text=_literal_as_label_reference)+        if partition_by is not None:+            self.partition_by = ClauseList(+                *util.to_list(partition_by),+                _literal_as_text=_literal_as_label_reference)++        if range_:+            self.range_ = self._interpret_range(range_)+            if rows:+                raise exc.ArgumentError(+                    "'range_' and 'rows' are mutually exclusive")+            else:+                self.rows = None+        elif rows:+            self.rows = self._interpret_range(rows)+            self.range_ = None+        else:+            self.rows = self.range_ = None++    def _interpret_range(self, range_):+        if not isinstance(range_, tuple) or len(range_) != 2:+            raise exc.ArgumentError("2-tuple expected for range/rows")++        if range_[0] is None:+            lower = RANGE_UNBOUNDED+        else:+            try:+                lower = int(range_[0])+            except ValueError:+                raise exc.ArgumentError(+                    "Integer or None expected for range value")+            else:+                if lower == 0:+                    lower = RANGE_CURRENT++        if range_[1] is None:+            upper = RANGE_UNBOUNDED+        else:+            try:+                upper = int(range_[1])+            except ValueError:+                raise exc.ArgumentError(+                    "Integer or None expected for range value")+            else:+                if upper == 0:+                    upper = RANGE_CURRENT++        return lower, upper++    @property+    def func(self):+        """the element referred to by this :class:`.Over`+        clause.++        .. deprecated:: 1.1 the ``func`` element has been renamed to+           ``.element``.  The two attributes are synonymous though+           ``.func`` is read-only.++        """+        return self.element++    @util.memoized_property+    def type(self):+        return self.element.type++    def get_children(self, **kwargs):+        return [c for c in+                (self.element, self.partition_by, self.order_by)+                if c is not None]++    def _copy_internals(self, clone=_clone, **kw):+        self.element = clone(self.element, **kw)+        if self.partition_by is not None:+            self.partition_by = clone(self.partition_by, **kw)+        if self.order_by is not None:+            self.order_by = clone(self.order_by, **kw)++    @property+    def _from_objects(self):+        return list(itertools.chain(+            *[c._from_objects for c in+                (self.element, self.partition_by, self.order_by)+              if c is not None]+        ))+++class WithinGroup(ColumnElement):+    """Represent a WITHIN GROUP (ORDER BY) clause.++    This is a special operator against so-called+    "ordered set aggregate" and "hypothetical+    set aggregate" functions, including ``percentile_cont()``,+    ``rank()``, ``dense_rank()``, etc.++    It's supported only by certain database backends, such as PostgreSQL,+    Oracle and MS SQL Server.++    The :class:`.WithinGroup` construct extracts its type from the+    method :meth:`.FunctionElement.within_group_type`.  If this returns+    ``None``, the function's ``.type`` is used.++    """+    __visit_name__ = 'withingroup'++    order_by = None++    def __init__(self, element, *order_by):+        r"""Produce a :class:`.WithinGroup` object against a function.++        Used against so-called "ordered set aggregate" and "hypothetical+        set aggregate" functions, including :class:`.percentile_cont`,+        :class:`.rank`, :class:`.dense_rank`, etc.++        :func:`~.expression.within_group` is usually called using+        the :meth:`.FunctionElement.within_group` method, e.g.::++            from sqlalchemy import within_group+            stmt = select([+                department.c.id,+                func.percentile_cont(0.5).within_group(+                    department.c.salary.desc()+                )+            ])++        The above statement would produce SQL similar to+        ``SELECT department.id, percentile_cont(0.5)+        WITHIN GROUP (ORDER BY department.salary DESC)``.++        :param element: a :class:`.FunctionElement` construct, typically+         generated by :data:`~.expression.func`.+        :param \*order_by: one or more column elements that will be used+         as the ORDER BY clause of the WITHIN GROUP construct.++        .. versionadded:: 1.1++        .. seealso::++            :data:`.expression.func`++            :func:`.expression.over`++        """+        self.element = element+        if order_by is not None:+            self.order_by = ClauseList(+                *util.to_list(order_by),+                _literal_as_text=_literal_as_label_reference)++    def over(self, partition_by=None, order_by=None):+        """Produce an OVER clause against this :class:`.WithinGroup`+        construct.++        This function has the same signature as that of+        :meth:`.FunctionElement.over`.++        """+        return Over(self, partition_by=partition_by, order_by=order_by)++    @util.memoized_property+    def type(self):+        wgt = self.element.within_group_type(self)+        if wgt is not None:+            return wgt+        else:+            return self.element.type++    def get_children(self, **kwargs):+        return [c for c in+                (self.element, self.order_by)+                if c is not None]++    def _copy_internals(self, clone=_clone, **kw):+        self.element = clone(self.element, **kw)+        if self.order_by is not None:+            self.order_by = clone(self.order_by, **kw)++    @property+    def _from_objects(self):+        return list(itertools.chain(+            *[c._from_objects for c in+                (self.element, self.order_by)+              if c is not None]+        ))+++class FunctionFilter(ColumnElement):+    """Represent a function FILTER clause.++    This is a special operator against aggregate and window functions,+    which controls which rows are passed to it.+    It's supported only by certain database backends.++    Invocation of :class:`.FunctionFilter` is via+    :meth:`.FunctionElement.filter`::++        func.count(1).filter(True)++    .. versionadded:: 1.0.0++    .. seealso::++        :meth:`.FunctionElement.filter`++    """+    __visit_name__ = 'funcfilter'++    criterion = None++    def __init__(self, func, *criterion):+        """Produce a :class:`.FunctionFilter` object against a function.++        Used against aggregate and window functions,+        for database backends that support the "FILTER" clause.++        E.g.::++            from sqlalchemy import funcfilter+            funcfilter(func.count(1), MyClass.name == 'some name')++        Would produce "COUNT(1) FILTER (WHERE myclass.name = 'some name')".++        This function is also available from the :data:`~.expression.func`+        construct itself via the :meth:`.FunctionElement.filter` method.++        .. versionadded:: 1.0.0++        .. seealso::++            :meth:`.FunctionElement.filter`+++        """+        self.func = func+        self.filter(*criterion)++    def filter(self, *criterion):+        """Produce an additional FILTER against the function.++        This method adds additional criteria to the initial criteria+        set up by :meth:`.FunctionElement.filter`.++        Multiple criteria are joined together at SQL render time+        via ``AND``.+++        """++        for criterion in list(criterion):+            criterion = _expression_literal_as_text(criterion)++            if self.criterion is not None:+                self.criterion = self.criterion & criterion+            else:+                self.criterion = criterion++        return self++    def over(self, partition_by=None, order_by=None):+        """Produce an OVER clause against this filtered function.++        Used against aggregate or so-called "window" functions,+        for database backends that support window functions.++        The expression::++            func.rank().filter(MyClass.y > 5).over(order_by='x')++        is shorthand for::++            from sqlalchemy import over, funcfilter+            over(funcfilter(func.rank(), MyClass.y > 5), order_by='x')++        See :func:`~.expression.over` for a full description.++        """+        return Over(self, partition_by=partition_by, order_by=order_by)++    @util.memoized_property+    def type(self):+        return self.func.type++    def get_children(self, **kwargs):+        return [c for c in+                (self.func, self.criterion)+                if c is not None]++    def _copy_internals(self, clone=_clone, **kw):+        self.func = clone(self.func, **kw)+        if self.criterion is not None:+            self.criterion = clone(self.criterion, **kw)++    @property+    def _from_objects(self):+        return list(itertools.chain(+            *[c._from_objects for c in (self.func, self.criterion)+              if c is not None]+        ))+++class Label(ColumnElement):+    """Represents a column label (AS).++    Represent a label, as typically applied to any column-level+    element using the ``AS`` sql keyword.++    """++    __visit_name__ = 'label'++    def __init__(self, name, element, type_=None):+        """Return a :class:`Label` object for the+        given :class:`.ColumnElement`.++        A label changes the name of an element in the columns clause of a+        ``SELECT`` statement, typically via the ``AS`` SQL keyword.++        This functionality is more conveniently available via the+        :meth:`.ColumnElement.label` method on :class:`.ColumnElement`.++        :param name: label name++        :param obj: a :class:`.ColumnElement`.++        """++        if isinstance(element, Label):+            self._resolve_label = element._label++        while isinstance(element, Label):+            element = element.element++        if name:+            self.name = name+            self._resolve_label = self.name+        else:+            self.name = _anonymous_label(+                '%%(%d %s)s' % (id(self), getattr(element, 'name', 'anon'))+            )++        self.key = self._label = self._key_label = self.name+        self._element = element+        self._type = type_+        self._proxies = [element]++    def __reduce__(self):+        return self.__class__, (self.name, self._element, self._type)++    @util.memoized_property+    def _allow_label_resolve(self):+        return self.element._allow_label_resolve++    @property+    def _order_by_label_element(self):+        return self++    @util.memoized_property+    def type(self):+        return type_api.to_instance(+            self._type or getattr(self._element, 'type', None)+        )++    @util.memoized_property+    def element(self):+        return self._element.self_group(against=operators.as_)++    def self_group(self, against=None):+        return self._apply_to_inner(self._element.self_group, against=against)++    def _negate(self):+        return self._apply_to_inner(self._element._negate)++    def _apply_to_inner(self, fn, *arg, **kw):+        sub_element = fn(*arg, **kw)+        if sub_element is not self._element:+            return Label(self.name,+                         sub_element,+                         type_=self._type)+        else:+            return self++    @property+    def primary_key(self):+        return self.element.primary_key++    @property+    def foreign_keys(self):+        return self.element.foreign_keys++    def get_children(self, **kwargs):+        return self.element,++    def _copy_internals(self, clone=_clone, anonymize_labels=False, **kw):+        self._element = clone(self._element, **kw)+        self.__dict__.pop('element', None)+        self.__dict__.pop('_allow_label_resolve', None)+        if anonymize_labels:+            self.name = self._resolve_label = _anonymous_label(+                '%%(%d %s)s' % (+                    id(self), getattr(self.element, 'name', 'anon'))+            )+            self.key = self._label = self._key_label = self.name++    @property+    def _from_objects(self):+        return self.element._from_objects++    def _make_proxy(self, selectable, name=None, **kw):+        e = self.element._make_proxy(selectable,+                                     name=name if name else self.name)+        e._proxies.append(self)+        if self._type is not None:+            e.type = self._type+        return e+++class ColumnClause(Immutable, ColumnElement):+    """Represents a column expression from any textual string.++    The :class:`.ColumnClause`, a lightweight analogue to the+    :class:`.Column` class, is typically invoked using the+    :func:`.column` function, as in::++        from sqlalchemy import column++        id, name = column("id"), column("name")+        stmt = select([id, name]).select_from("user")++    The above statement would produce SQL like::++        SELECT id, name FROM user++    :class:`.ColumnClause` is the immediate superclass of the schema-specific+    :class:`.Column` object.  While the :class:`.Column` class has all the+    same capabilities as :class:`.ColumnClause`, the :class:`.ColumnClause`+    class is usable by itself in those cases where behavioral requirements+    are limited to simple SQL expression generation.  The object has none of+    the associations with schema-level metadata or with execution-time+    behavior that :class:`.Column` does, so in that sense is a "lightweight"+    version of :class:`.Column`.++    Full details on :class:`.ColumnClause` usage is at :func:`.column`.++    .. seealso::++        :func:`.column`++        :class:`.Column`++    """+    __visit_name__ = 'column'++    onupdate = default = server_default = server_onupdate = None++    _is_multiparam_column = False++    _memoized_property = util.group_expirable_memoized_property()++    def __init__(self, text, type_=None, is_literal=False, _selectable=None):+        """Produce a :class:`.ColumnClause` object.++        The :class:`.ColumnClause` is a lightweight analogue to the+        :class:`.Column` class.  The :func:`.column` function can+        be invoked with just a name alone, as in::++            from sqlalchemy import column++            id, name = column("id"), column("name")+            stmt = select([id, name]).select_from("user")++        The above statement would produce SQL like::++            SELECT id, name FROM user++        Once constructed, :func:`.column` may be used like any other SQL+        expression element such as within :func:`.select` constructs::++            from sqlalchemy.sql import column++            id, name = column("id"), column("name")+            stmt = select([id, name]).select_from("user")++        The text handled by :func:`.column` is assumed to be handled+        like the name of a database column; if the string contains mixed case,+        special characters, or matches a known reserved word on the target+        backend, the column expression will render using the quoting+        behavior determined by the backend.  To produce a textual SQL+        expression that is rendered exactly without any quoting,+        use :func:`.literal_column` instead, or pass ``True`` as the+        value of :paramref:`.column.is_literal`.   Additionally, full SQL+        statements are best handled using the :func:`.text` construct.++        :func:`.column` can be used in a table-like+        fashion by combining it with the :func:`.table` function+        (which is the lightweight analogue to :class:`.Table`) to produce+        a working table construct with minimal boilerplate::++            from sqlalchemy import table, column, select++            user = table("user",+                    column("id"),+                    column("name"),+                    column("description"),+            )++            stmt = select([user.c.description]).where(user.c.name == 'wendy')++        A :func:`.column` / :func:`.table` construct like that illustrated+        above can be created in an+        ad-hoc fashion and is not associated with any+        :class:`.schema.MetaData`, DDL, or events, unlike its+        :class:`.Table` counterpart.++        .. versionchanged:: 1.0.0 :func:`.expression.column` can now+           be imported from the plain ``sqlalchemy`` namespace like any+           other SQL element.++        :param text: the text of the element.++        :param type: :class:`.types.TypeEngine` object which can associate+          this :class:`.ColumnClause` with a type.++        :param is_literal: if True, the :class:`.ColumnClause` is assumed to+          be an exact expression that will be delivered to the output with no+          quoting rules applied regardless of case sensitive settings. the+          :func:`.literal_column()` function essentially invokes+          :func:`.column` while passing ``is_literal=True``.++        .. seealso::++            :class:`.Column`++            :func:`.literal_column`++            :func:`.table`++            :func:`.text`++            :ref:`sqlexpression_literal_column`++        """++        self.key = self.name = text+        self.table = _selectable+        self.type = type_api.to_instance(type_)+        self.is_literal = is_literal++    def _compare_name_for_result(self, other):+        if self.is_literal or \+                self.table is None or self.table._textual or \+                not hasattr(other, 'proxy_set') or (+                    isinstance(other, ColumnClause) and+                    (other.is_literal or+                     other.table is None or+                     other.table._textual)+                ):+            return (hasattr(other, 'name') and self.name == other.name) or \+                (hasattr(other, '_label') and self._label == other._label)+        else:+            return other.proxy_set.intersection(self.proxy_set)++    def _get_table(self):+        return self.__dict__['table']++    def _set_table(self, table):+        self._memoized_property.expire_instance(self)+        self.__dict__['table'] = table+    table = property(_get_table, _set_table)++    @_memoized_property+    def _from_objects(self):+        t = self.table+        if t is not None:+            return [t]+        else:+            return []++    @util.memoized_property+    def description(self):+        if util.py3k:+            return self.name+        else:+            return self.name.encode('ascii', 'backslashreplace')++    @_memoized_property+    def _key_label(self):+        if self.key != self.name:+            return self._gen_label(self.key)+        else:+            return self._label++    @_memoized_property+    def _label(self):+        return self._gen_label(self.name)++    @_memoized_property+    def _render_label_in_columns_clause(self):+        return self.table is not None++    def _gen_label(self, name):+        t = self.table++        if self.is_literal:+            return None++        elif t is not None and t.named_with_column:+            if getattr(t, 'schema', None):+                label = t.schema.replace('.', '_') + "_" + \+                    t.name + "_" + name+            else:+                label = t.name + "_" + name++            # propagate name quoting rules for labels.+            if getattr(name, "quote", None) is not None:+                if isinstance(label, quoted_name):+                    label.quote = name.quote+                else:+                    label = quoted_name(label, name.quote)+            elif getattr(t.name, "quote", None) is not None:+                # can't get this situation to occur, so let's+                # assert false on it for now+                assert not isinstance(label, quoted_name)+                label = quoted_name(label, t.name.quote)++            # ensure the label name doesn't conflict with that+            # of an existing column+            if label in t.c:+                _label = label+                counter = 1+                while _label in t.c:+                    _label = label + "_" + str(counter)+                    counter += 1+                label = _label++            return _as_truncated(label)++        else:+            return name++    def _bind_param(self, operator, obj, type_=None):+        return BindParameter(self.key, obj,+                             _compared_to_operator=operator,+                             _compared_to_type=self.type,+                             type_=type_,+                             unique=True)++    def _make_proxy(self, selectable, name=None, attach=True,+                    name_is_truncatable=False, **kw):+        # propagate the "is_literal" flag only if we are keeping our name,+        # otherwise its considered to be a label+        is_literal = self.is_literal and (name is None or name == self.name)+        c = self._constructor(+            _as_truncated(name or self.name) if+            name_is_truncatable else+            (name or self.name),+            type_=self.type,+            _selectable=selectable,+            is_literal=is_literal+        )+        if name is None:+            c.key = self.key+        c._proxies = [self]+        if selectable._is_clone_of is not None:+            c._is_clone_of = \+                selectable._is_clone_of.columns.get(c.key)++        if attach:+            selectable._columns[c.key] = c+        return c+++class CollationClause(ColumnElement):+    __visit_name__ = "collation"++    def __init__(self, collation):+        self.collation = collation+++class _IdentifiedClause(Executable, ClauseElement):++    __visit_name__ = 'identified'+    _execution_options = \+        Executable._execution_options.union({'autocommit': False})++    def __init__(self, ident):+        self.ident = ident+++class SavepointClause(_IdentifiedClause):+    __visit_name__ = 'savepoint'+++class RollbackToSavepointClause(_IdentifiedClause):+    __visit_name__ = 'rollback_to_savepoint'+++class ReleaseSavepointClause(_IdentifiedClause):+    __visit_name__ = 'release_savepoint'+++class quoted_name(util.MemoizedSlots, util.text_type):+    """Represent a SQL identifier combined with quoting preferences.++    :class:`.quoted_name` is a Python unicode/str subclass which+    represents a particular identifier name along with a+    ``quote`` flag.  This ``quote`` flag, when set to+    ``True`` or ``False``, overrides automatic quoting behavior+    for this identifier in order to either unconditionally quote+    or to not quote the name.  If left at its default of ``None``,+    quoting behavior is applied to the identifier on a per-backend basis+    based on an examination of the token itself.++    A :class:`.quoted_name` object with ``quote=True`` is also+    prevented from being modified in the case of a so-called+    "name normalize" option.  Certain database backends, such as+    Oracle, Firebird, and DB2 "normalize" case-insensitive names+    as uppercase.  The SQLAlchemy dialects for these backends+    convert from SQLAlchemy's lower-case-means-insensitive convention+    to the upper-case-means-insensitive conventions of those backends.+    The ``quote=True`` flag here will prevent this conversion from occurring+    to support an identifier that's quoted as all lower case against+    such a backend.++    The :class:`.quoted_name` object is normally created automatically+    when specifying the name for key schema constructs such as+    :class:`.Table`, :class:`.Column`, and others.  The class can also be+    passed explicitly as the name to any function that receives a name which+    can be quoted.  Such as to use the :meth:`.Engine.has_table` method with+    an unconditionally quoted name::++        from sqlalchemy import create_engine+        from sqlalchemy.sql import quoted_name++        engine = create_engine("oracle+cx_oracle://some_dsn")+        engine.has_table(quoted_name("some_table", True))++    The above logic will run the "has table" logic against the Oracle backend,+    passing the name exactly as ``"some_table"`` without converting to+    upper case.++    .. versionadded:: 0.9.0++    .. versionchanged:: 1.2 The :class:`.quoted_name` construct is now+       importable from ``sqlalchemy.sql``, in addition to the previous+       location of ``sqlalchemy.sql.elements``.++    """++    __slots__ = 'quote', 'lower', 'upper'++    def __new__(cls, value, quote):+        if value is None:+            return None+        # experimental - don't bother with quoted_name+        # if quote flag is None.  doesn't seem to make any dent+        # in performance however+        # elif not sprcls and quote is None:+        #   return value+        elif isinstance(value, cls) and (+            quote is None or value.quote == quote+        ):+            return value+        self = super(quoted_name, cls).__new__(cls, value)+        self.quote = quote+        return self++    def __reduce__(self):+        return quoted_name, (util.text_type(self), self.quote)++    def _memoized_method_lower(self):+        if self.quote:+            return self+        else:+            return util.text_type(self).lower()++    def _memoized_method_upper(self):+        if self.quote:+            return self+        else:+            return util.text_type(self).upper()++    def __repr__(self):+        backslashed = self.encode('ascii', 'backslashreplace')+        if not util.py2k:+            backslashed = backslashed.decode('ascii')+        return "'%s'" % backslashed+++class _truncated_label(quoted_name):+    """A unicode subclass used to identify symbolic "+    "names that may require truncation."""++    __slots__ = ()++    def __new__(cls, value, quote=None):+        quote = getattr(value, "quote", quote)+        # return super(_truncated_label, cls).__new__(cls, value, quote, True)+        return super(_truncated_label, cls).__new__(cls, value, quote)++    def __reduce__(self):+        return self.__class__, (util.text_type(self), self.quote)++    def apply_map(self, map_):+        return self+++class conv(_truncated_label):+    """Mark a string indicating that a name has already been converted+    by a naming convention.++    This is a string subclass that indicates a name that should not be+    subject to any further naming conventions.++    E.g. when we create a :class:`.Constraint` using a naming convention+    as follows::++        m = MetaData(naming_convention={+            "ck": "ck_%(table_name)s_%(constraint_name)s"+        })+        t = Table('t', m, Column('x', Integer),+                        CheckConstraint('x > 5', name='x5'))++    The name of the above constraint will be rendered as ``"ck_t_x5"``.+    That is, the existing name ``x5`` is used in the naming convention as the+    ``constraint_name`` token.++    In some situations, such as in migration scripts, we may be rendering+    the above :class:`.CheckConstraint` with a name that's already been+    converted.  In order to make sure the name isn't double-modified, the+    new name is applied using the :func:`.schema.conv` marker.  We can+    use this explicitly as follows::+++        m = MetaData(naming_convention={+            "ck": "ck_%(table_name)s_%(constraint_name)s"+        })+        t = Table('t', m, Column('x', Integer),+                        CheckConstraint('x > 5', name=conv('ck_t_x5')))++    Where above, the :func:`.schema.conv` marker indicates that the constraint+    name here is final, and the name will render as ``"ck_t_x5"`` and not+    ``"ck_t_ck_t_x5"``++    .. versionadded:: 0.9.4++    .. seealso::++        :ref:`constraint_naming_conventions`++    """+    __slots__ = ()+++class _defer_name(_truncated_label):+    """mark a name as 'deferred' for the purposes of automated name+    generation.++    """+    __slots__ = ()++    def __new__(cls, value):+        if value is None:+            return _NONE_NAME+        elif isinstance(value, conv):+            return value+        else:+            return super(_defer_name, cls).__new__(cls, value)++    def __reduce__(self):+        return self.__class__, (util.text_type(self), )+++class _defer_none_name(_defer_name):+    """indicate a 'deferred' name that was ultimately the value None."""+    __slots__ = ()++_NONE_NAME = _defer_none_name("_unnamed_")++# for backwards compatibility in case+# someone is re-implementing the+# _truncated_identifier() sequence in a custom+# compiler+_generated_label = _truncated_label+++class _anonymous_label(_truncated_label):+    """A unicode subclass used to identify anonymously+    generated names."""++    __slots__ = ()++    def __add__(self, other):+        return _anonymous_label(+            quoted_name(+                util.text_type.__add__(self, util.text_type(other)),+                self.quote)+        )++    def __radd__(self, other):+        return _anonymous_label(+            quoted_name(+                util.text_type.__add__(util.text_type(other), self),+                self.quote)+        )++    def apply_map(self, map_):+        if self.quote is not None:+            # preserve quoting only if necessary+            return quoted_name(self % map_, self.quote)+        else:+            # else skip the constructor call+            return self % map_+++def _as_truncated(value):+    """coerce the given value to :class:`._truncated_label`.++    Existing :class:`._truncated_label` and+    :class:`._anonymous_label` objects are passed+    unchanged.+    """++    if isinstance(value, _truncated_label):+        return value+    else:+        return _truncated_label(value)+++def _string_or_unprintable(element):+    if isinstance(element, util.string_types):+        return element+    else:+        try:+            return str(element)+        except Exception:+            return "unprintable element %r" % element+++def _expand_cloned(elements):+    """expand the given set of ClauseElements to be the set of all 'cloned'+    predecessors.++    """+    return itertools.chain(*[x._cloned_set for x in elements])+++def _select_iterables(elements):+    """expand tables into individual columns in the+    given list of column expressions.++    """+    return itertools.chain(*[c._select_iterable for c in elements])+++def _cloned_intersection(a, b):+    """return the intersection of sets a and b, counting+    any overlap between 'cloned' predecessors.++    The returned set is in terms of the entities present within 'a'.++    """+    all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b))+    return set(elem for elem in a+               if all_overlap.intersection(elem._cloned_set))+++def _cloned_difference(a, b):+    all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b))+    return set(elem for elem in a+               if not all_overlap.intersection(elem._cloned_set))+++@util.dependencies("sqlalchemy.sql.functions")+def _labeled(functions, element):+    if not hasattr(element, 'name') or \+            isinstance(element, functions.FunctionElement):+        return element.label(None)+    else:+        return element+++def _is_column(col):+    """True if ``col`` is an instance of :class:`.ColumnElement`."""++    return isinstance(col, ColumnElement)+++def _find_columns(clause):+    """locate Column objects within the given expression."""++    cols = util.column_set()+    traverse(clause, {}, {'column': cols.add})+    return cols+++# there is some inconsistency here between the usage of+# inspect() vs. checking for Visitable and __clause_element__.+# Ideally all functions here would derive from inspect(),+# however the inspect() versions add significant callcount+# overhead for critical functions like _interpret_as_column_or_from().+# Generally, the column-based functions are more performance critical+# and are fine just checking for __clause_element__().  It is only+# _interpret_as_from() where we'd like to be able to receive ORM entities+# that have no defined namespace, hence inspect() is needed there.+++def _column_as_key(element):+    if isinstance(element, util.string_types):+        return element+    if hasattr(element, '__clause_element__'):+        element = element.__clause_element__()+    try:+        return element.key+    except AttributeError:+        return None+++def _clause_element_as_expr(element):+    if hasattr(element, '__clause_element__'):+        return element.__clause_element__()+    else:+        return element+++def _literal_as_label_reference(element):+    if isinstance(element, util.string_types):+        return _textual_label_reference(element)++    elif hasattr(element, '__clause_element__'):+        element = element.__clause_element__()++    return _literal_as_text(element)+++def _literal_and_labels_as_label_reference(element):+    if isinstance(element, util.string_types):+        return _textual_label_reference(element)++    elif hasattr(element, '__clause_element__'):+        element = element.__clause_element__()++    if isinstance(element, ColumnElement) and \+            element._order_by_label_element is not None:+        return _label_reference(element)+    else:+        return _literal_as_text(element)+++def _expression_literal_as_text(element):+    return _literal_as_text(element, warn=True)+++def _literal_as_text(element, warn=False):+    if isinstance(element, Visitable):+        return element+    elif hasattr(element, '__clause_element__'):+        return element.__clause_element__()+    elif isinstance(element, util.string_types):+        if warn:+            util.warn_limited(+                "Textual SQL expression %(expr)r should be "+                "explicitly declared as text(%(expr)r)",+                {"expr": util.ellipses_string(element)})++        return TextClause(util.text_type(element))+    elif isinstance(element, (util.NoneType, bool)):+        return _const_expr(element)+    else:+        raise exc.ArgumentError(+            "SQL expression object or string expected, got object of type %r "+            "instead" % type(element)+        )+++def _no_literals(element):+    if hasattr(element, '__clause_element__'):+        return element.__clause_element__()+    elif not isinstance(element, Visitable):+        raise exc.ArgumentError("Ambiguous literal: %r.  Use the 'text()' "+                                "function to indicate a SQL expression "+                                "literal, or 'literal()' to indicate a "+                                "bound value." % (element, ))+    else:+        return element+++def _is_literal(element):+    return not isinstance(element, Visitable) and \+        not hasattr(element, '__clause_element__')+++def _only_column_elements_or_none(element, name):+    if element is None:+        return None+    else:+        return _only_column_elements(element, name)+++def _only_column_elements(element, name):+    if hasattr(element, '__clause_element__'):+        element = element.__clause_element__()+    if not isinstance(element, ColumnElement):+        raise exc.ArgumentError(+            "Column-based expression object expected for argument "+            "'%s'; got: '%s', type %s" % (name, element, type(element)))+    return element+++def _literal_as_binds(element, name=None, type_=None):+    if hasattr(element, '__clause_element__'):+        return element.__clause_element__()+    elif not isinstance(element, Visitable):+        if element is None:+            return Null()+        else:+            return BindParameter(name, element, type_=type_, unique=True)+    else:+        return element++_guess_straight_column = re.compile(r'^\w\S*$', re.I)+++def _interpret_as_column_or_from(element):+    if isinstance(element, Visitable):+        return element+    elif hasattr(element, '__clause_element__'):+        return element.__clause_element__()++    insp = inspection.inspect(element, raiseerr=False)+    if insp is None:+        if isinstance(element, (util.NoneType, bool)):+            return _const_expr(element)+    elif hasattr(insp, "selectable"):+        return insp.selectable++    # be forgiving as this is an extremely common+    # and known expression+    if element == "*":+        guess_is_literal = True+    elif isinstance(element, (numbers.Number)):+        return ColumnClause(str(element), is_literal=True)+    else:+        element = str(element)+        # give into temptation, as this fact we are guessing about+        # is not one we've previously ever needed our users tell us;+        # but let them know we are not happy about it+        guess_is_literal = not _guess_straight_column.match(element)+        util.warn_limited(+            "Textual column expression %(column)r should be "+            "explicitly declared with text(%(column)r), "+            "or use %(literal_column)s(%(column)r) "+            "for more specificity",+            {+                "column": util.ellipses_string(element),+                "literal_column": "literal_column"+                if guess_is_literal else "column"+            })+    return ColumnClause(+        element,+        is_literal=guess_is_literal)+++def _const_expr(element):+    if isinstance(element, (Null, False_, True_)):+        return element+    elif element is None:+        return Null()+    elif element is False:+        return False_()+    elif element is True:+        return True_()+    else:+        raise exc.ArgumentError(+            "Expected None, False, or True"+        )+++def _type_from_args(args):+    for a in args:+        if not a.type._isnull:+            return a.type+    else:+        return type_api.NULLTYPE+++def _corresponding_column_or_error(fromclause, column,+                                   require_embedded=False):+    c = fromclause.corresponding_column(column,+                                        require_embedded=require_embedded)+    if c is None:+        raise exc.InvalidRequestError(+            "Given column '%s', attached to table '%s', "+            "failed to locate a corresponding column from table '%s'"+            %+            (column,+             getattr(column, 'table', None),+             fromclause.description)+        )+    return c+++class AnnotatedColumnElement(Annotated):+    def __init__(self, element, values):+        Annotated.__init__(self, element, values)+        ColumnElement.comparator._reset(self)+        for attr in ('name', 'key', 'table'):+            if self.__dict__.get(attr, False) is None:+                self.__dict__.pop(attr)++    def _with_annotations(self, values):+        clone = super(AnnotatedColumnElement, self)._with_annotations(values)+        ColumnElement.comparator._reset(clone)+        return clone++    @util.memoized_property+    def name(self):+        """pull 'name' from parent, if not present"""+        return self._Annotated__element.name++    @util.memoized_property+    def table(self):+        """pull 'table' from parent, if not present"""+        return self._Annotated__element.table++    @util.memoized_property+    def key(self):+        """pull 'key' from parent, if not present"""+        return self._Annotated__element.key++    @util.memoized_property+    def info(self):+        return self._Annotated__element.info++    @util.memoized_property+    def anon_label(self):+        return self._Annotated__element.anon_label
+ test/files/string.py view
@@ -0,0 +1,43 @@+DATETIME_FORMAT = r'j. E Y \k\e\l\l\o G.i'++# This string contains \r followed by \n+"""
+"""++latex_preamble = r'''+\usepackage{amsmath}+\DeclareUnicodeCharacter{00A0}{\nobreakspace}+% In the parameters section, place a newline after the Parameters+% header+\usepackage{expdlist}+\let\latexdescription=\description+\def\description{\latexdescription{}{} \breaklabel}+% Make Examples/etc section headers smaller and more compact+\makeatletter+\titleformat{\paragraph}{\normalsize\py@HeaderFamily}%+            {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor}+\titlespacing*{\paragraph}{0pt}{1ex}{0pt}+\makeatother+% Fix footer/header+\renewcommand{\chaptermark}[1]{\markboth{\MakeUppercase{\thechapter.\ #1}}{}}+\renewcommand{\sectionmark}[1]{\markright{\MakeUppercase{\thesection.\ #1}}}+'''++def literal(value, type_=None):+    r"""Return a literal clause, bound to a bind parameter.++    Literal clauses are created automatically when non-+    :class:`.ClauseElement` objects (such as strings, ints, dates, etc.) are+    used in a comparison operation with a :class:`.ColumnElement` subclass,+    such as a :class:`~sqlalchemy.schema.Column` object.  Use this function+    to force the generation of a literal clause, which will be created as a+    :class:`BindParameter` with a bound value.++    :param value: the value to be bound. Can be any Python object supported by+        the underlying DB-API, or is translatable via the given type argument.++    :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` which+        will provide bind-parameter translation for this literal.++    """+    return BindParameter(None, value, type_=type_, unique=True)
+ test/files/test.py view
@@ -0,0 +1,16 @@+from blah import  boo+import baz   as wop++def thing():+    pass++def    hello():+    what; up;++def boo(a, *b, c=1, **d):+    pass++def bar(a=1, *b):+    f(a=1, *b)++a = [b for c in d if e == f]
+ test/files/typeann.py view
@@ -0,0 +1,23 @@+def a(b:c):+    pass++def a(b:c=d):+    pass++def a(*b:c):+    pass++def a(**b:c):+    pass++def a(b : c):+    pass++def a(b : c = d):+    pass++def a(*b : c):+    pass++def a(**b : c):+    pass
+ test/files/weird.py view
@@ -0,0 +1,4 @@+(   1+         *+  3+    )
+ test/files/weird2.py view
@@ -0,0 +1,17 @@+\+ while False:+ pass++[()for a in()if not(yield)]++[(yield)for a in()]++def a():+    pass  # cmt++def a():a=yield++def a(+ b=None+):+  pass