inline-python 0.2 → 0.2.0.1
raw patch · 9 files changed
+303/−23 lines, 9 filesdep ~quickcheck-instancesdep ~vector
Dependency ranges changed: quickcheck-instances, vector
Files
- ChangeLog.md +9/−1
- cbits/python.c +70/−0
- include/inline-python.h +30/−0
- inline-python.cabal +12/−4
- src/Python/Inline.hs +2/−1
- src/Python/Inline/Literal.hs +125/−11
- test/TST/FromPy.hs +22/−0
- test/TST/Roundtrip.hs +5/−6
- test/TST/ToPy.hs +28/−0
ChangeLog.md view
@@ -1,7 +1,15 @@+0.2.1.0 [2026.01.13]+----------------+* `From/ToPy` instance for `Integer`&`Natural` added.+* `vector-0.13.2` is required.+* Python>=3.10 is supported. Boolean marshaling with python<3.12 is+ fixed. Previously it caused crashes on counter decrement.+* Documentation fixes.+ 0.2 [2025.05.04] ---------------- * `FromPy`/`ToPy` instances added for: `Complex`, both strict and lazy `Text` &- `ByteString`, `ShortByteString`, `Maybe a`+ `ByteString`, `ShortByteString`, `Maybe a`. * Module `Python.Inline.Eval` added which support for eval/exec with user supplied global and local variables. * QuasiQuotes `Python.Inline.QQ.pycode` added for creating `PyQuote` data type.
cbits/python.c view
@@ -1,6 +1,9 @@ #include <inline-python.h> #include <stdlib.h> +#include "MachDeps.h"++ // ================================================================ // Callbacks //@@ -140,3 +143,70 @@ return -1; } ++PyObject* inline_py_Integer_ToPy(+ void* buf,+ size_t size,+ int sign+ )+{+ PyObject* num =+#if PY_MINOR_VERSION < 13+ _PyLong_FromByteArray(buf, size,+ 1, // Little endian+ 0 // Unsigned+ );+#else+ PyLong_FromNativeBytes(buf, size,+ Py_ASNATIVEBYTES_LITTLE_ENDIAN |+ Py_ASNATIVEBYTES_UNSIGNED_BUFFER+ );+#endif+ if( sign ) {+ PyObject* neg = PyNumber_Negative(num);+ Py_DECREF(num);+ return neg;+ } else {+ return num;+ }+}+++ssize_t inline_py_Long_ByteSize(PyObject* p) {+ // See NOTE: [Integer encoding/decoding]+ //+ // PyLong_AsNativeBytes allows to compute buffer size but it does+ // so according to python's memory layout+#if WORD_SIZE_IN_BITS == 32+ const int shiftW = 2;+#elif WORD_SIZE_IN_BITS == 64+ const int shiftW = 3;+#else+#error "Something wrong with MachDeps.h"+#endif+ const int shift = shiftW + 3;+ const ssize_t mask = (1<<shift) - 1;+ const ssize_t bits = _PyLong_NumBits(p);+ if( bits & mask ) {+ return ((bits >> shift) + 1) << shiftW;+ } else {+ return (bits >> shift) << shiftW;+ }+}++void inline_py_Integer_FromPy(+ PyObject* p,+ void* buf,+ size_t size+ )+{+ // N.B. _PyLong_AsByteArray changed signature in 3.13+#if PY_MINOR_VERSION < 13+ _PyLong_AsByteArray((PyLongObject*)p, buf, size,+ 1, // little_endian+ 0 // is_signed+ );+#else+ PyLong_AsNativeBytes(p, buf, size, -1);+#endif+}
include/inline-python.h view
@@ -51,3 +51,33 @@ int n, PyObject **out );++// Python's C API only gained public function to create integers of+// arbitrary size in 3.13. We have to use internals for earlier+// versions.+PyObject* inline_py_Integer_ToPy(+ void* buf, // Buffer holding number + size_t size, // Buffer size in bytes+ int sign // Sign of number (0 is +, 1 is -)+);++// Compute size of buffer which can hold decoded number +// and satistfy Integer's requirements+//+// See: NOTE: [Integer encoding/decoding]+//+// PRECONDITION: parameter must instance of PyLong. This is not+// checked.+// PRECONDITION: passed number must be positive+ssize_t inline_py_Long_ByteSize(PyObject* p);++// Parse python integral number into buffer. This is compatibility+// shim.+//+// PRECONDITION: parameter must instance of PyLong. This is not+// checked.+void inline_py_Integer_FromPy(+ PyObject* p,+ void* buf,+ size_t size+);
inline-python.cabal view
@@ -2,7 +2,7 @@ Build-Type: Simple Name: inline-python-Version: 0.2+Version: 0.2.0.1 Synopsis: Python interpreter embedded into haskell. Description: This package embeds python interpreter into haskell program and@@ -23,6 +23,14 @@ include/inline-python.h py/bound-vars.py +Tested-With:+ GHC == 9.2.8+ GHC == 9.4.8+ GHC == 9.6.7+ GHC == 9.8.4+ GHC == 9.10.2+ GHC == 9.12.2+ source-repository head type: git location: http://github.com/Shimuuar/inline-python@@ -59,12 +67,12 @@ , text >=2 , bytestring >=0.11.2 , exceptions >=0.10- , vector >=0.13+ , vector >=0.13.2 hs-source-dirs: src include-dirs: include c-sources: cbits/python.c cc-options: -g -Wall- pkgconfig-depends: python3-embed+ pkgconfig-depends: python3-embed >= 3.10 -- Exposed-modules: Python.Inline@@ -90,7 +98,7 @@ , tasty >=1.2 , tasty-hunit >=0.10 , tasty-quickcheck >=0.10- , quickcheck-instances >=0.3.32+ , quickcheck-instances >=0.3.33 , exceptions , containers , vector
src/Python/Inline.hs view
@@ -2,6 +2,7 @@ -- python code in haskell programs. Take for example following program: -- -- > {-# LANGUAGE QuasiQuotes #-}+-- > import Control.Monad -- > import Python.Inline -- > import Python.Inline.QQ -- >@@ -10,7 +11,7 @@ -- > let input = [1..10] :: [Int] -- > let square :: Int -> Py Int -- > square x = pure (x * x)--- > print =<< runPy $ do+-- > print <=< runPy $ do -- > fromPy' @[Int] =<< [pye| [ square_hs(x) for x in input_hs ] |] -- -- Quasiquotation 'Python.Inline.QQ.pye' captures variables @input@
src/Python/Inline/Literal.hs view
@@ -1,8 +1,8 @@-{-# LANGUAGE CPP #-} {-# LANGUAGE ForeignFunctionInterface #-} {-# LANGUAGE MagicHash #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE TemplateHaskell #-}+{-# LANGUAGE UnliftedFFITypes #-} -- | -- Conversion between haskell data types and python values module Python.Inline.Literal@@ -34,18 +34,22 @@ import Data.Vector.Generic qualified as VG import Data.Vector.Generic.Mutable qualified as MVG import Data.Vector qualified as V-#if MIN_VERSION_vector(0,13,2) import Data.Vector.Strict qualified as VV-#endif import Data.Vector.Storable qualified as VS import Data.Vector.Primitive qualified as VP import Data.Vector.Unboxed qualified as VU+import Data.Primitive.ByteArray qualified as BA+import Data.Primitive.Types (Prim(..))+import Numeric.Natural (Natural) import Foreign.Ptr import Foreign.C.Types import Foreign.Storable import Foreign.Marshal.Alloc (alloca,mallocBytes) import Foreign.Marshal.Utils (copyBytes) import GHC.Float (float2Double, double2Float)+import GHC.Exts (Int(..),Word(..),sizeofByteArray#,ByteArray#)+import GHC.Num.Natural qualified+import GHC.Num.Integer qualified import Data.Complex (Complex((:+))) import Language.C.Inline qualified as C@@ -290,6 +294,121 @@ | otherwise -> throwM OutOfRange ++-- NOTE: [Integer encoding/decoding]+-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+--+-- Interfacing between arbitrary precision integers in haskell and+-- python is pain: they have different representations. And python got+-- API for working with large numbers only in 3.13. We have to use+-- internal API for earlier versions.+--+-- Only large number are discussed below. Small are straightforward+-- enough.+--+-- + GHC's Integer use sign + little endian sequence of Word#. Since+-- all supported platforms are LE it's same as little endian+-- sequence of bytes.+--+-- + Important invariant: highest word must be nonzero!+--+-- + Python uses two-complement.+--+-- One problem is computation of required buffer size. (8byte word is+-- assumed). For example 2^63 requires 9 bytes in two-complement+-- encoding since we need one bit for sign. But 8 bytes enough for+-- Integer's encoding. Sign is stored separately.+++-- | @since 0.2.1.0+instance ToPy Integer where+ basicToPy (GHC.Num.Integer.IS i) = basicToPy (I# i)+ basicToPy (GHC.Num.Integer.IP p) = Py $ do+ let n = fromIntegral (I# (sizeofByteArray# p)) :: CSize+ inline_py_Integer_ToPy p n 0+ basicToPy (GHC.Num.Integer.IN p) = Py $ do+ let n = fromIntegral (I# (sizeofByteArray# p)) :: CSize+ inline_py_Integer_ToPy p n 1++-- | @since 0.2.1.0+instance ToPy Natural where+ basicToPy (GHC.Num.Natural.NS i) = basicToPy (W# i)+ basicToPy (GHC.Num.Natural.NB p) = Py $ do+ let n = fromIntegral (I# (sizeofByteArray# p)) :: CSize+ inline_py_Integer_ToPy p n 0++-- | @since 0.2.1.0+instance FromPy Integer where+ basicFromPy p = runProgram $ do+ progIO [CU.exp| int { PyLong_Check($(PyObject *p)) } |] >>= \case+ 0 -> progIO $ throwM BadPyType+ _ -> pure ()+ -- At this point we know that p is number+ p_overflow <- withPyAlloca+ n <- progIO [CU.exp| long long { PyLong_AsLongLongAndOverflow($(PyObject* p), $(int* p_overflow)) } |]+ progIO (peek p_overflow) >>= \case+ -- Number fits into long long+ 0 -> return $! fromIntegral n+ -- Number is positive+ 1 -> do+ BA.ByteArray ba <- progIO $ decodePositiveInteger p+ pure $ GHC.Num.Integer.IP ba+ -- Number is negative+ -1 -> do+ neg <- takeOwnership+ <=< progPy+ $ throwOnNULL =<< Py [CU.exp| PyObject* { PyNumber_Negative( $(PyObject *p) ) } |]+ BA.ByteArray ba <- progIO $ decodePositiveInteger neg+ pure $ GHC.Num.Integer.IN ba+ -- Unreachable+ _ -> error "inline-py: FromPy Integer: INTERNAL ERROR"+ where++-- | @since 0.2.1.0+instance FromPy Natural where+ basicFromPy p = runProgram $ do+ progIO [CU.exp| int { PyLong_Check($(PyObject *p)) } |] >>= \case+ 0 -> progIO $ throwM BadPyType+ _ -> pure ()+ p_overflow <- withPyAlloca+ n <- progIO [CU.exp| long long { PyLong_AsLongLongAndOverflow($(PyObject* p), $(int* p_overflow)) } |]+ progIO (peek p_overflow) >>= \case+ -- Number fits into long long+ 0 | n < 0 -> progIO $ throwM OutOfRange+ | otherwise -> return $! fromIntegral n+ -- Number is negative+ -1 -> progIO $ throwM OutOfRange+ -- Number is positive.+ --+ -- NOTE that if size of bytearray is equal to size of word we+ -- need to return small constructor+ 1 -> progIO $ decodePositiveInteger p >>= \case+ BA.ByteArray ba+ | I# (sizeofByteArray# ba) == (finiteBitSize (0::Word) `div` 8)+ -> pure $! case indexByteArray# ba 0# of+ W# w -> GHC.Num.Natural.NS w+ | otherwise+ -> pure $! GHC.Num.Natural.NB ba+ -- Unreachable+ _ -> error "inline-py: FromPy Natural: INTERNAL ERROR"++-- Decode large positive number:+-- + Must be instance of PyLong+-- + Must be positive+decodePositiveInteger :: Ptr PyObject -> IO BA.ByteArray+decodePositiveInteger p_num = do+ sz <- [CU.exp| int { inline_py_Long_ByteSize( $(PyObject *p_num) ) } |]+ buf@(BA.MutableByteArray ptr_buf) <- BA.newByteArray (fromIntegral sz)+ _ <- inline_py_Integer_FromPy p_num ptr_buf (fromIntegral sz)+ BA.unsafeFreezeByteArray buf++++foreign import ccall unsafe "inline_py_Integer_ToPy"+ inline_py_Integer_ToPy :: ByteArray# -> CSize -> CInt -> IO (Ptr PyObject)+foreign import ccall unsafe "inline_py_Integer_FromPy"+ inline_py_Integer_FromPy :: Ptr PyObject -> BA.MutableByteArray# MVG.RealWorld -> CSize -> IO CInt+ -- | Encoded as 1-character string instance ToPy Char where basicToPy c = do@@ -308,8 +427,7 @@ r <- Py [CU.block| int { PyObject* p = $(PyObject *p); if( !PyUnicode_Check(p) )- return -1;- if( 1 != PyUnicode_GET_LENGTH(p) )+ return -1; if( 1 != PyUnicode_GET_LENGTH(p) ) return -1; switch( PyUnicode_KIND(p) ) { case PyUnicode_1BYTE_KIND:@@ -325,8 +443,8 @@ | otherwise -> pure $ chr $ fromIntegral r instance ToPy Bool where- basicToPy True = Py [CU.exp| PyObject* { Py_True } |]- basicToPy False = Py [CU.exp| PyObject* { Py_False } |]+ basicToPy True = Py [CU.block| PyObject* { Py_RETURN_TRUE; } |]+ basicToPy False = Py [CU.block| PyObject* { Py_RETURN_FALSE; } |] -- | Uses python's truthiness conventions instance FromPy Bool where@@ -515,11 +633,9 @@ -- | Converts to python's list instance (ToPy a, VU.Unbox a) => ToPy (VU.Vector a) where basicToPy = vectorToPy-#if MIN_VERSION_vector(0,13,2) -- | Converts to python's list instance (ToPy a) => ToPy (VV.Vector a) where basicToPy = vectorToPy-#endif -- | Accepts python's sequence (@len@ and indexing) instance FromPy a => FromPy (V.Vector a) where@@ -533,11 +649,9 @@ -- | Accepts python's sequence (@len@ and indexing) instance (FromPy a, VU.Unbox a) => FromPy (VU.Vector a) where basicFromPy = vectorFromPy-#if MIN_VERSION_vector(0,13,2) -- | Accepts python's sequence (@len@ and indexing) instance FromPy a => FromPy (VV.Vector a) where basicFromPy = vectorFromPy-#endif -- | Fold over python's iterator. Function takes ownership over iterator.
test/TST/FromPy.hs view
@@ -10,6 +10,7 @@ import Python.Inline import Python.Inline.QQ import Data.Complex (Complex((:+)))+import Numeric.Natural (Natural) import TST.Util @@ -92,6 +93,27 @@ , testCase "[3]" $ eq @[Int] (Just [1,2,3]) [pye| [1,2,3] |] , testCase "Int" $ eq @[Int] Nothing [pye| None |] ]+ , testGroup "Integer" $+ let eqI = eq @Integer . Just+ in concat+ [ [ testCase (" 2^"++show k++"-1") $ eqI (2^k - 1) [pye| 2**k_hs - 1 |]+ , testCase (" 2^"++show k) $ eqI (2^k ) [pye| 2**k_hs |]+ , testCase (" 2^"++show k++"+1") $ eqI (2^k + 1) [pye| 2**k_hs + 1 |]+ , testCase ("-2^"++show k++"-1") $ eqI (negate $ 2^k - 1) [pye| -(2**k_hs - 1) |]+ , testCase ("-2^"++show k) $ eqI (negate $ 2^k ) [pye| -(2**k_hs) |]+ , testCase ("-2^"++show k++"+1") $ eqI (negate $ 2^k + 1) [pye| -(2**k_hs + 1) |]+ ]+ | k <- [63,64,65,92,17,128,129,32100] :: [Int]+ ]+ , testGroup "Natural" $+ let eqI = eq @Natural . Just+ in concat+ [ [ testCase (" 2^"++show k++"-1") $ eqI (2^k - 1) [pye| 2**k_hs - 1 |]+ , testCase (" 2^"++show k) $ eqI (2^k ) [pye| 2**k_hs |]+ , testCase (" 2^"++show k++"+1") $ eqI (2^k + 1) [pye| 2**k_hs + 1 |]+ ]+ | k <- [63,64,65,92,17,128,129,32100] :: [Int]+ ] ] failE :: forall a. (Eq a, Show a, FromPy a) => PyObject -> Py ()
test/TST/Roundtrip.hs view
@@ -1,5 +1,4 @@ {-# LANGUAGE AllowAmbiguousTypes #-}-{-# LANGUAGE CPP #-} -- | module TST.Roundtrip (tests) where @@ -12,12 +11,14 @@ import Data.Text.Lazy qualified as TL import Data.Complex (Complex) import Foreign.C.Types+import Numeric.Natural (Natural) import Test.Tasty import Test.Tasty.QuickCheck import Test.QuickCheck.Instances.Vector () import Test.QuickCheck.Instances.ByteString () import Test.QuickCheck.Instances.Text ()+import Test.QuickCheck.Instances.Natural () import Python.Inline import Python.Inline.QQ @@ -25,9 +26,7 @@ import Data.ByteString.Lazy qualified as BL import Data.ByteString.Short qualified as SBS import Data.Vector qualified as V-#if MIN_VERSION_vector(0,13,2) import Data.Vector.Strict qualified as VV-#endif import Data.Vector.Storable qualified as VS import Data.Vector.Primitive qualified as VP import Data.Vector.Unboxed qualified as VU@@ -47,6 +46,8 @@ , testRoundtrip @Word32 , testRoundtrip @Word64 , testRoundtrip @Word+ , testRoundtrip @Integer+ , testRoundtrip @Natural -- C wrappers , testRoundtrip @CChar , testRoundtrip @CSChar@@ -85,9 +86,7 @@ , testRoundtrip @(VS.Vector Int) , testRoundtrip @(VP.Vector Int) , testRoundtrip @(VU.Vector Int)-#if MIN_VERSION_vector(0,13,2)--- , testRoundtrip @(VV.Vector Int)-#endif+ , testRoundtrip @(VV.Vector Int) , testRoundtrip @BS.ByteString , testRoundtrip @BL.ByteString , testRoundtrip @SBS.ShortByteString
test/TST/ToPy.hs view
@@ -6,6 +6,7 @@ import Data.Set qualified as Set import Data.Map.Strict qualified as Map import Data.Complex (Complex((:+)))+import Numeric.Natural (Natural) import Test.Tasty import Test.Tasty.HUnit import Python.Inline@@ -57,4 +58,31 @@ , testCase "dict unhashable" $ runPy $ let x = Map.fromList [([1],10), ([5],50), ([3],30)] :: Map.Map [Int] Int in throwsPy [py_| x_hs |]+ -- Integer. We only check large number here. Small will be well tested by QC+ , testGroup "Integer" $ concat+ [ [ testCase (" 2^"++show k++"-1") $ let n = 2^k - 1 :: Integer+ in runPy [py_| assert n_hs == 2**k_hs - 1 |]+ , testCase (" 2^"++show k) $ let n = 2^k :: Integer+ in runPy [py_| assert n_hs == 2**k_hs |]+ , testCase (" 2^"++show k++"+1") $ let n = 2^k + 1 :: Integer+ in runPy [py_| assert n_hs == 2**k_hs + 1 |]+ , testCase ("-2^"++show k++"-1") $ let n = negate $ 2^k - 1 :: Integer+ in runPy [py_| assert n_hs == -(2**k_hs - 1) |]+ , testCase ("-2^"++show k) $ let n = negate $ 2^k :: Integer+ in runPy [py_| assert n_hs == -(2**k_hs) |]+ , testCase ("-2^"++show k++"+1") $ let n = negate $ 2^k + 1 :: Integer+ in runPy [py_|assert n_hs == -(2**k_hs + 1) |]+ ]+ | k <- [63,64,65,92,17,128,129,32100] :: [Int]+ ]+ , testGroup "Natural" $ concat+ [ [ testCase (" 2^"++show k++"-1") $ let n = 2^k - 1 :: Natural+ in runPy [py_| assert n_hs == 2**k_hs - 1 |]+ , testCase (" 2^"++show k) $ let n = 2^k :: Natural+ in runPy [py_| assert n_hs == 2**k_hs |]+ , testCase (" 2^"++show k++"+1") $ let n = 2^k + 1 :: Natural+ in runPy [py_| assert n_hs == 2**k_hs + 1 |]+ ]+ | k <- [63,64,65,92,17,128,129,32100] :: [Int]+ ] ]