diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,7 +1,15 @@ +0.2.1.0 [2026.01.13] +---------------- +* `From/ToPy` instance for `Integer`&`Natural` added. +* `vector-0.13.2` is required. +* Python>=3.10 is supported. Boolean marshaling with python<3.12 is + fixed. Previously it caused crashes on counter decrement. +* Documentation fixes. + 0.2 [2025.05.04] ---------------- * `FromPy`/`ToPy` instances added for: `Complex`, both strict and lazy `Text` & - `ByteString`, `ShortByteString`, `Maybe a` + `ByteString`, `ShortByteString`, `Maybe a`. * Module `Python.Inline.Eval` added which support for eval/exec with user supplied global and local variables. * QuasiQuotes `Python.Inline.QQ.pycode` added for creating `PyQuote` data type. diff --git a/cbits/python.c b/cbits/python.c --- a/cbits/python.c +++ b/cbits/python.c @@ -1,6 +1,9 @@ #include #include +#include "MachDeps.h" + + // ================================================================ // Callbacks // @@ -140,3 +143,70 @@ return -1; } + +PyObject* inline_py_Integer_ToPy( + void* buf, + size_t size, + int sign + ) +{ + PyObject* num = +#if PY_MINOR_VERSION < 13 + _PyLong_FromByteArray(buf, size, + 1, // Little endian + 0 // Unsigned + ); +#else + PyLong_FromNativeBytes(buf, size, + Py_ASNATIVEBYTES_LITTLE_ENDIAN | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER + ); +#endif + if( sign ) { + PyObject* neg = PyNumber_Negative(num); + Py_DECREF(num); + return neg; + } else { + return num; + } +} + + +ssize_t inline_py_Long_ByteSize(PyObject* p) { + // See NOTE: [Integer encoding/decoding] + // + // PyLong_AsNativeBytes allows to compute buffer size but it does + // so according to python's memory layout +#if WORD_SIZE_IN_BITS == 32 + const int shiftW = 2; +#elif WORD_SIZE_IN_BITS == 64 + const int shiftW = 3; +#else +#error "Something wrong with MachDeps.h" +#endif + const int shift = shiftW + 3; + const ssize_t mask = (1<> shift) + 1) << shiftW; + } else { + return (bits >> shift) << shiftW; + } +} + +void inline_py_Integer_FromPy( + PyObject* p, + void* buf, + size_t size + ) +{ + // N.B. _PyLong_AsByteArray changed signature in 3.13 +#if PY_MINOR_VERSION < 13 + _PyLong_AsByteArray((PyLongObject*)p, buf, size, + 1, // little_endian + 0 // is_signed + ); +#else + PyLong_AsNativeBytes(p, buf, size, -1); +#endif +} diff --git a/include/inline-python.h b/include/inline-python.h --- a/include/inline-python.h +++ b/include/inline-python.h @@ -51,3 +51,33 @@ int n, PyObject **out ); + +// Python's C API only gained public function to create integers of +// arbitrary size in 3.13. We have to use internals for earlier +// versions. +PyObject* inline_py_Integer_ToPy( + void* buf, // Buffer holding number + size_t size, // Buffer size in bytes + int sign // Sign of number (0 is +, 1 is -) +); + +// Compute size of buffer which can hold decoded number +// and satistfy Integer's requirements +// +// See: NOTE: [Integer encoding/decoding] +// +// PRECONDITION: parameter must instance of PyLong. This is not +// checked. +// PRECONDITION: passed number must be positive +ssize_t inline_py_Long_ByteSize(PyObject* p); + +// Parse python integral number into buffer. This is compatibility +// shim. +// +// PRECONDITION: parameter must instance of PyLong. This is not +// checked. +void inline_py_Integer_FromPy( + PyObject* p, + void* buf, + size_t size +); diff --git a/inline-python.cabal b/inline-python.cabal --- a/inline-python.cabal +++ b/inline-python.cabal @@ -2,7 +2,7 @@ Build-Type: Simple Name: inline-python -Version: 0.2 +Version: 0.2.0.1 Synopsis: Python interpreter embedded into haskell. Description: This package embeds python interpreter into haskell program and @@ -23,6 +23,14 @@ include/inline-python.h py/bound-vars.py +Tested-With: + GHC == 9.2.8 + GHC == 9.4.8 + GHC == 9.6.7 + GHC == 9.8.4 + GHC == 9.10.2 + GHC == 9.12.2 + source-repository head type: git location: http://github.com/Shimuuar/inline-python @@ -59,12 +67,12 @@ , text >=2 , bytestring >=0.11.2 , exceptions >=0.10 - , vector >=0.13 + , vector >=0.13.2 hs-source-dirs: src include-dirs: include c-sources: cbits/python.c cc-options: -g -Wall - pkgconfig-depends: python3-embed + pkgconfig-depends: python3-embed >= 3.10 -- Exposed-modules: Python.Inline @@ -90,7 +98,7 @@ , tasty >=1.2 , tasty-hunit >=0.10 , tasty-quickcheck >=0.10 - , quickcheck-instances >=0.3.32 + , quickcheck-instances >=0.3.33 , exceptions , containers , vector diff --git a/src/Python/Inline.hs b/src/Python/Inline.hs --- a/src/Python/Inline.hs +++ b/src/Python/Inline.hs @@ -2,6 +2,7 @@ -- python code in haskell programs. Take for example following program: -- -- > {-# LANGUAGE QuasiQuotes #-} +-- > import Control.Monad -- > import Python.Inline -- > import Python.Inline.QQ -- > @@ -10,7 +11,7 @@ -- > let input = [1..10] :: [Int] -- > let square :: Int -> Py Int -- > square x = pure (x * x) --- > print =<< runPy $ do +-- > print <=< runPy $ do -- > fromPy' @[Int] =<< [pye| [ square_hs(x) for x in input_hs ] |] -- -- Quasiquotation 'Python.Inline.QQ.pye' captures variables @input@ diff --git a/src/Python/Inline/Literal.hs b/src/Python/Inline/Literal.hs --- a/src/Python/Inline/Literal.hs +++ b/src/Python/Inline/Literal.hs @@ -1,8 +1,8 @@ -{-# LANGUAGE CPP #-} {-# LANGUAGE ForeignFunctionInterface #-} {-# LANGUAGE MagicHash #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE UnliftedFFITypes #-} -- | -- Conversion between haskell data types and python values module Python.Inline.Literal @@ -34,18 +34,22 @@ import Data.Vector.Generic qualified as VG import Data.Vector.Generic.Mutable qualified as MVG import Data.Vector qualified as V -#if MIN_VERSION_vector(0,13,2) import Data.Vector.Strict qualified as VV -#endif import Data.Vector.Storable qualified as VS import Data.Vector.Primitive qualified as VP import Data.Vector.Unboxed qualified as VU +import Data.Primitive.ByteArray qualified as BA +import Data.Primitive.Types (Prim(..)) +import Numeric.Natural (Natural) import Foreign.Ptr import Foreign.C.Types import Foreign.Storable import Foreign.Marshal.Alloc (alloca,mallocBytes) import Foreign.Marshal.Utils (copyBytes) import GHC.Float (float2Double, double2Float) +import GHC.Exts (Int(..),Word(..),sizeofByteArray#,ByteArray#) +import GHC.Num.Natural qualified +import GHC.Num.Integer qualified import Data.Complex (Complex((:+))) import Language.C.Inline qualified as C @@ -290,6 +294,121 @@ | otherwise -> throwM OutOfRange + +-- NOTE: [Integer encoding/decoding] +-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-- +-- Interfacing between arbitrary precision integers in haskell and +-- python is pain: they have different representations. And python got +-- API for working with large numbers only in 3.13. We have to use +-- internal API for earlier versions. +-- +-- Only large number are discussed below. Small are straightforward +-- enough. +-- +-- + GHC's Integer use sign + little endian sequence of Word#. Since +-- all supported platforms are LE it's same as little endian +-- sequence of bytes. +-- +-- + Important invariant: highest word must be nonzero! +-- +-- + Python uses two-complement. +-- +-- One problem is computation of required buffer size. (8byte word is +-- assumed). For example 2^63 requires 9 bytes in two-complement +-- encoding since we need one bit for sign. But 8 bytes enough for +-- Integer's encoding. Sign is stored separately. + + +-- | @since 0.2.1.0 +instance ToPy Integer where + basicToPy (GHC.Num.Integer.IS i) = basicToPy (I# i) + basicToPy (GHC.Num.Integer.IP p) = Py $ do + let n = fromIntegral (I# (sizeofByteArray# p)) :: CSize + inline_py_Integer_ToPy p n 0 + basicToPy (GHC.Num.Integer.IN p) = Py $ do + let n = fromIntegral (I# (sizeofByteArray# p)) :: CSize + inline_py_Integer_ToPy p n 1 + +-- | @since 0.2.1.0 +instance ToPy Natural where + basicToPy (GHC.Num.Natural.NS i) = basicToPy (W# i) + basicToPy (GHC.Num.Natural.NB p) = Py $ do + let n = fromIntegral (I# (sizeofByteArray# p)) :: CSize + inline_py_Integer_ToPy p n 0 + +-- | @since 0.2.1.0 +instance FromPy Integer where + basicFromPy p = runProgram $ do + progIO [CU.exp| int { PyLong_Check($(PyObject *p)) } |] >>= \case + 0 -> progIO $ throwM BadPyType + _ -> pure () + -- At this point we know that p is number + p_overflow <- withPyAlloca + n <- progIO [CU.exp| long long { PyLong_AsLongLongAndOverflow($(PyObject* p), $(int* p_overflow)) } |] + progIO (peek p_overflow) >>= \case + -- Number fits into long long + 0 -> return $! fromIntegral n + -- Number is positive + 1 -> do + BA.ByteArray ba <- progIO $ decodePositiveInteger p + pure $ GHC.Num.Integer.IP ba + -- Number is negative + -1 -> do + neg <- takeOwnership + <=< progPy + $ throwOnNULL =<< Py [CU.exp| PyObject* { PyNumber_Negative( $(PyObject *p) ) } |] + BA.ByteArray ba <- progIO $ decodePositiveInteger neg + pure $ GHC.Num.Integer.IN ba + -- Unreachable + _ -> error "inline-py: FromPy Integer: INTERNAL ERROR" + where + +-- | @since 0.2.1.0 +instance FromPy Natural where + basicFromPy p = runProgram $ do + progIO [CU.exp| int { PyLong_Check($(PyObject *p)) } |] >>= \case + 0 -> progIO $ throwM BadPyType + _ -> pure () + p_overflow <- withPyAlloca + n <- progIO [CU.exp| long long { PyLong_AsLongLongAndOverflow($(PyObject* p), $(int* p_overflow)) } |] + progIO (peek p_overflow) >>= \case + -- Number fits into long long + 0 | n < 0 -> progIO $ throwM OutOfRange + | otherwise -> return $! fromIntegral n + -- Number is negative + -1 -> progIO $ throwM OutOfRange + -- Number is positive. + -- + -- NOTE that if size of bytearray is equal to size of word we + -- need to return small constructor + 1 -> progIO $ decodePositiveInteger p >>= \case + BA.ByteArray ba + | I# (sizeofByteArray# ba) == (finiteBitSize (0::Word) `div` 8) + -> pure $! case indexByteArray# ba 0# of + W# w -> GHC.Num.Natural.NS w + | otherwise + -> pure $! GHC.Num.Natural.NB ba + -- Unreachable + _ -> error "inline-py: FromPy Natural: INTERNAL ERROR" + +-- Decode large positive number: +-- + Must be instance of PyLong +-- + Must be positive +decodePositiveInteger :: Ptr PyObject -> IO BA.ByteArray +decodePositiveInteger p_num = do + sz <- [CU.exp| int { inline_py_Long_ByteSize( $(PyObject *p_num) ) } |] + buf@(BA.MutableByteArray ptr_buf) <- BA.newByteArray (fromIntegral sz) + _ <- inline_py_Integer_FromPy p_num ptr_buf (fromIntegral sz) + BA.unsafeFreezeByteArray buf + + + +foreign import ccall unsafe "inline_py_Integer_ToPy" + inline_py_Integer_ToPy :: ByteArray# -> CSize -> CInt -> IO (Ptr PyObject) +foreign import ccall unsafe "inline_py_Integer_FromPy" + inline_py_Integer_FromPy :: Ptr PyObject -> BA.MutableByteArray# MVG.RealWorld -> CSize -> IO CInt + -- | Encoded as 1-character string instance ToPy Char where basicToPy c = do @@ -308,8 +427,7 @@ r <- Py [CU.block| int { PyObject* p = $(PyObject *p); if( !PyUnicode_Check(p) ) - return -1; - if( 1 != PyUnicode_GET_LENGTH(p) ) + return -1; if( 1 != PyUnicode_GET_LENGTH(p) ) return -1; switch( PyUnicode_KIND(p) ) { case PyUnicode_1BYTE_KIND: @@ -325,8 +443,8 @@ | otherwise -> pure $ chr $ fromIntegral r instance ToPy Bool where - basicToPy True = Py [CU.exp| PyObject* { Py_True } |] - basicToPy False = Py [CU.exp| PyObject* { Py_False } |] + basicToPy True = Py [CU.block| PyObject* { Py_RETURN_TRUE; } |] + basicToPy False = Py [CU.block| PyObject* { Py_RETURN_FALSE; } |] -- | Uses python's truthiness conventions instance FromPy Bool where @@ -515,11 +633,9 @@ -- | Converts to python's list instance (ToPy a, VU.Unbox a) => ToPy (VU.Vector a) where basicToPy = vectorToPy -#if MIN_VERSION_vector(0,13,2) -- | Converts to python's list instance (ToPy a) => ToPy (VV.Vector a) where basicToPy = vectorToPy -#endif -- | Accepts python's sequence (@len@ and indexing) instance FromPy a => FromPy (V.Vector a) where @@ -533,11 +649,9 @@ -- | Accepts python's sequence (@len@ and indexing) instance (FromPy a, VU.Unbox a) => FromPy (VU.Vector a) where basicFromPy = vectorFromPy -#if MIN_VERSION_vector(0,13,2) -- | Accepts python's sequence (@len@ and indexing) instance FromPy a => FromPy (VV.Vector a) where basicFromPy = vectorFromPy -#endif -- | Fold over python's iterator. Function takes ownership over iterator. diff --git a/test/TST/FromPy.hs b/test/TST/FromPy.hs --- a/test/TST/FromPy.hs +++ b/test/TST/FromPy.hs @@ -10,6 +10,7 @@ import Python.Inline import Python.Inline.QQ import Data.Complex (Complex((:+))) +import Numeric.Natural (Natural) import TST.Util @@ -92,6 +93,27 @@ , testCase "[3]" $ eq @[Int] (Just [1,2,3]) [pye| [1,2,3] |] , testCase "Int" $ eq @[Int] Nothing [pye| None |] ] + , testGroup "Integer" $ + let eqI = eq @Integer . Just + in concat + [ [ testCase (" 2^"++show k++"-1") $ eqI (2^k - 1) [pye| 2**k_hs - 1 |] + , testCase (" 2^"++show k) $ eqI (2^k ) [pye| 2**k_hs |] + , testCase (" 2^"++show k++"+1") $ eqI (2^k + 1) [pye| 2**k_hs + 1 |] + , testCase ("-2^"++show k++"-1") $ eqI (negate $ 2^k - 1) [pye| -(2**k_hs - 1) |] + , testCase ("-2^"++show k) $ eqI (negate $ 2^k ) [pye| -(2**k_hs) |] + , testCase ("-2^"++show k++"+1") $ eqI (negate $ 2^k + 1) [pye| -(2**k_hs + 1) |] + ] + | k <- [63,64,65,92,17,128,129,32100] :: [Int] + ] + , testGroup "Natural" $ + let eqI = eq @Natural . Just + in concat + [ [ testCase (" 2^"++show k++"-1") $ eqI (2^k - 1) [pye| 2**k_hs - 1 |] + , testCase (" 2^"++show k) $ eqI (2^k ) [pye| 2**k_hs |] + , testCase (" 2^"++show k++"+1") $ eqI (2^k + 1) [pye| 2**k_hs + 1 |] + ] + | k <- [63,64,65,92,17,128,129,32100] :: [Int] + ] ] failE :: forall a. (Eq a, Show a, FromPy a) => PyObject -> Py () diff --git a/test/TST/Roundtrip.hs b/test/TST/Roundtrip.hs --- a/test/TST/Roundtrip.hs +++ b/test/TST/Roundtrip.hs @@ -1,5 +1,4 @@ {-# LANGUAGE AllowAmbiguousTypes #-} -{-# LANGUAGE CPP #-} -- | module TST.Roundtrip (tests) where @@ -12,12 +11,14 @@ import Data.Text.Lazy qualified as TL import Data.Complex (Complex) import Foreign.C.Types +import Numeric.Natural (Natural) import Test.Tasty import Test.Tasty.QuickCheck import Test.QuickCheck.Instances.Vector () import Test.QuickCheck.Instances.ByteString () import Test.QuickCheck.Instances.Text () +import Test.QuickCheck.Instances.Natural () import Python.Inline import Python.Inline.QQ @@ -25,9 +26,7 @@ import Data.ByteString.Lazy qualified as BL import Data.ByteString.Short qualified as SBS import Data.Vector qualified as V -#if MIN_VERSION_vector(0,13,2) import Data.Vector.Strict qualified as VV -#endif import Data.Vector.Storable qualified as VS import Data.Vector.Primitive qualified as VP import Data.Vector.Unboxed qualified as VU @@ -47,6 +46,8 @@ , testRoundtrip @Word32 , testRoundtrip @Word64 , testRoundtrip @Word + , testRoundtrip @Integer + , testRoundtrip @Natural -- C wrappers , testRoundtrip @CChar , testRoundtrip @CSChar @@ -85,9 +86,7 @@ , testRoundtrip @(VS.Vector Int) , testRoundtrip @(VP.Vector Int) , testRoundtrip @(VU.Vector Int) -#if MIN_VERSION_vector(0,13,2) --- , testRoundtrip @(VV.Vector Int) -#endif + , testRoundtrip @(VV.Vector Int) , testRoundtrip @BS.ByteString , testRoundtrip @BL.ByteString , testRoundtrip @SBS.ShortByteString diff --git a/test/TST/ToPy.hs b/test/TST/ToPy.hs --- a/test/TST/ToPy.hs +++ b/test/TST/ToPy.hs @@ -6,6 +6,7 @@ import Data.Set qualified as Set import Data.Map.Strict qualified as Map import Data.Complex (Complex((:+))) +import Numeric.Natural (Natural) import Test.Tasty import Test.Tasty.HUnit import Python.Inline @@ -57,4 +58,31 @@ , testCase "dict unhashable" $ runPy $ let x = Map.fromList [([1],10), ([5],50), ([3],30)] :: Map.Map [Int] Int in throwsPy [py_| x_hs |] + -- Integer. We only check large number here. Small will be well tested by QC + , testGroup "Integer" $ concat + [ [ testCase (" 2^"++show k++"-1") $ let n = 2^k - 1 :: Integer + in runPy [py_| assert n_hs == 2**k_hs - 1 |] + , testCase (" 2^"++show k) $ let n = 2^k :: Integer + in runPy [py_| assert n_hs == 2**k_hs |] + , testCase (" 2^"++show k++"+1") $ let n = 2^k + 1 :: Integer + in runPy [py_| assert n_hs == 2**k_hs + 1 |] + , testCase ("-2^"++show k++"-1") $ let n = negate $ 2^k - 1 :: Integer + in runPy [py_| assert n_hs == -(2**k_hs - 1) |] + , testCase ("-2^"++show k) $ let n = negate $ 2^k :: Integer + in runPy [py_| assert n_hs == -(2**k_hs) |] + , testCase ("-2^"++show k++"+1") $ let n = negate $ 2^k + 1 :: Integer + in runPy [py_|assert n_hs == -(2**k_hs + 1) |] + ] + | k <- [63,64,65,92,17,128,129,32100] :: [Int] + ] + , testGroup "Natural" $ concat + [ [ testCase (" 2^"++show k++"-1") $ let n = 2^k - 1 :: Natural + in runPy [py_| assert n_hs == 2**k_hs - 1 |] + , testCase (" 2^"++show k) $ let n = 2^k :: Natural + in runPy [py_| assert n_hs == 2**k_hs |] + , testCase (" 2^"++show k++"+1") $ let n = 2^k + 1 :: Natural + in runPy [py_| assert n_hs == 2**k_hs + 1 |] + ] + | k <- [63,64,65,92,17,128,129,32100] :: [Int] + ] ]