packages feed

duckdb-ffi-2.0.0.0: test/ArrowInterfaceTest.hs

{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE TypeApplications #-}

module ArrowInterfaceTest (tests) where

import Control.Exception (bracket, finally)
import Control.Monad (when)
import Data.Bits (testBit)
import Data.Coerce (coerce)
import Data.Int (Int32)
import Data.Proxy (Proxy (..))
import Database.DuckDB.FFI
import Foreign.C.ConstPtr (ConstPtr (..))
import Foreign.C.String (peekCString)
import Foreign.C.Types (CChar)
import Foreign.Marshal.Alloc (alloca)
import Foreign.Marshal.Array (peekArray, withArray)
import Foreign.Marshal.Utils (withMany)
import Foreign.Ptr (Ptr, freeHaskellFunPtr, nullFunPtr, nullPtr)
import Foreign.Storable (Storable (..), peek, peekElemOff, poke, pokeElemOff)
import GHC.Records (getField)
import HsBindgen.Runtime.HasCField (fromPtr)
import HsBindgen.Runtime.Support.FunPtr (fromFunPtr, toFunPtr)
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=))
import Utils (releaseArrowArray, releaseArrowSchema, releaseArrowStream, withConnection, withConstCString, withDatabase)

tests :: TestTree
tests =
  testGroup
    "Arrow Interface"
    [ arrowSchemaRoundtrip
    , arrowChunkRoundtrip
    , arrowStreamErrorCallbacks
    ]

-- | Invoke a producer's error callback after its next-array callback fails.
arrowStreamErrorCallbacks :: TestTree
arrowStreamErrorCallbacks =
  testCase "stream callbacks report a producer error" $
    withConstCString "Arrow producer failed" \message ->
      bracket (toFunPtr (\_ _ -> pure 5)) (freeHaskellFunPtr . coerce) \next ->
        bracket (toFunPtr (const (pure message))) (freeHaskellFunPtr . coerce) \lastError ->
          bracket
            ( toFunPtr \stream -> do
                poke (fromPtr (Proxy @"release") (stream :: Ptr ArrowArrayStream)) nullFunPtr
            )
            (freeHaskellFunPtr . coerce)
            \release ->
              alloca \stream -> do
                poke stream (ArrowArrayStream nullFunPtr next lastError release nullPtr)
                alloca \array -> do
                  poke array zeroArrowArray
                  fromFunPtr next stream array >>= (@?= 5)
                  fromFunPtr lastError stream >>= (peekCString . coerce) >>= (@?= "Arrow producer failed")
                releaseArrowStream stream
                value <- peek stream
                (getField @"release") value @?= nullFunPtr
                releaseArrowStream stream

arrowSchemaRoundtrip :: TestTree
arrowSchemaRoundtrip =
  testCase "to_arrow_schema exposes children and converts back" $
    withDatabase \db ->
      withConnection db \conn ->
        withArrowOptions conn \arrowOpts ->
          withLogicalTypes [DuckDBTypeInteger, DuckDBTypeVarchar] \logicalTypes ->
            withArray logicalTypes \logicalArray ->
              withColumnNames ["id", "label"] \nameArray ->
                withArrowSchema \schemaPtr -> do
                  errData <- c_duckdb_to_arrow_schema arrowOpts logicalArray nameArray (fromIntegral (length logicalTypes)) (coerce schemaPtr)
                  assertNoError errData

                  schema <- peek schemaPtr
                  formatStr <- (peekCString . coerce) ((getField @"format") schema)
                  formatStr @?= "+s"
                  (getField @"n_children") schema @?= fromIntegral (length logicalTypes)

                  let childCount = fromIntegral ((getField @"n_children") schema)
                  let childArrayPtr = (getField @"children") schema
                  assertBool "children pointer should not be null" (childArrayPtr /= nullPtr)
                  [fc, sc] <- peekArray childCount childArrayPtr >>= mapM peek
                  (peekCString . coerce) ((getField @"name") fc) >>= (@?= "id")
                  (peekCString . coerce) ((getField @"name") sc) >>= (@?= "label")
                  assertBool "schema release pointer should be set" ((getField @"release") schema /= nullFunPtr)

                  withConvertedSchema conn schemaPtr (const (pure ()))

arrowChunkRoundtrip :: TestTree
arrowChunkRoundtrip =
  testCase "data_chunk_to_arrow and from_arrow preserve values and nulls" $
    withDatabase \db ->
      withConnection db \conn ->
        withArrowOptions conn \arrowOpts ->
          withLogicalTypes [DuckDBTypeInteger] \logicalTypes ->
            withArray logicalTypes \logicalArray ->
              withColumnNames ["val"] \nameArray ->
                withArrowSchema \schemaPtr -> do
                  errSchema <- c_duckdb_to_arrow_schema arrowOpts logicalArray nameArray 1 (coerce schemaPtr)
                  assertNoError errSchema

                  withConvertedSchema conn schemaPtr \convertedSchema ->
                    do
                      chunk <- c_duckdb_create_data_chunk logicalArray 1
                      assertBool "create_data_chunk should return chunk" (chunk /= coerce nullPtr)

                      withOwnedChunk chunk \ownedChunk -> do
                        c_duckdb_data_chunk_set_size ownedChunk 2
                        vector <- c_duckdb_data_chunk_get_vector ownedChunk 0
                        dataPtr <- c_duckdb_vector_get_data vector
                        let intPtr = coerce dataPtr :: Ptr Int32
                        pokeElemOff intPtr 0 42
                        pokeElemOff intPtr 1 0

                        c_duckdb_vector_ensure_validity_writable vector
                        maskPtr <- c_duckdb_vector_get_validity vector
                        assertBool "validity mask pointer should not be null" (maskPtr /= nullPtr)
                        c_duckdb_validity_set_row_valid maskPtr 0
                        c_duckdb_validity_set_row_invalid maskPtr 1

                        withArrowArray \arrayPtr ->
                          do
                            errArray <- c_duckdb_data_chunk_to_arrow arrowOpts ownedChunk (coerce arrayPtr)
                            assertNoError errArray

                            array <- peek arrayPtr
                            (getField @"length") array @?= 2
                            assertBool "release pointer should not be null before transfer" ((getField @"release") array /= nullFunPtr)

                            alloca \outChunkPtr -> do
                              poke outChunkPtr (coerce nullPtr)
                              errFromArrow <- c_duckdb_data_chunk_from_arrow conn (coerce arrayPtr) convertedSchema outChunkPtr
                              assertNoError errFromArrow
                              restoredChunk <- peek outChunkPtr
                              assertBool "restored chunk should not be null" (restoredChunk /= coerce nullPtr)

                              withOwnedChunk restoredChunk $ \restored -> do
                                restoredSize <- c_duckdb_data_chunk_get_size restored
                                restoredSize @?= 2

                                restoredVector <- c_duckdb_data_chunk_get_vector restored 0
                                restoredDataPtr <- c_duckdb_vector_get_data restoredVector
                                let restoredIntPtr = coerce restoredDataPtr :: Ptr Int32
                                restoredVal <- peekElemOff restoredIntPtr 0
                                restoredVal @?= 42

                                restoredMaskPtr <- c_duckdb_vector_get_validity restoredVector
                                assertBool "restored validity mask pointer should not be null" (restoredMaskPtr /= nullPtr)
                                restoredMaskWord <- peek restoredMaskPtr
                                assertBool "first row should be valid" (testBit restoredMaskWord 0)
                                assertBool "second row should be null" (not (testBit restoredMaskWord 1))

                              arrayAfter <- peek arrayPtr
                              (getField @"release") arrayAfter @?= nullFunPtr

withArrowOptions :: DuckDBConnection -> (DuckDBArrowOptions -> IO a) -> IO a
withArrowOptions conn action =
  alloca \optsPtr -> do
    let acquire = do
          poke optsPtr (coerce nullPtr)
          c_duckdb_connection_get_arrow_options conn optsPtr
          opts <- peek optsPtr
          when (opts == coerce nullPtr) $ assertFailure "duckdb_connection_get_arrow_options returned null"
          pure opts
        release _ = c_duckdb_destroy_arrow_options optsPtr
    bracket acquire release action

withLogicalTypes :: [DuckDBType] -> ([DuckDBLogicalType] -> IO a) -> IO a
withLogicalTypes [] action = action []
withLogicalTypes (t : ts) action =
  bracket (c_duckdb_create_logical_type (DuckDBTypeAlias t)) destroyLogicalType \lt ->
    withLogicalTypes ts \rest -> action (lt : rest)

withColumnNames :: [String] -> (Ptr (ConstPtr CChar) -> IO a) -> IO a
withColumnNames names action =
  withMany withConstCString names $ \cNames -> withArray cNames action

withArrowSchema :: (Ptr ArrowSchema -> IO a) -> IO a
withArrowSchema action =
  withStruct zeroArrowSchema \ptr -> action ptr `finally` releaseArrowSchema ptr

withArrowArray :: (Ptr ArrowArray -> IO a) -> IO a
withArrowArray action =
  withStruct zeroArrowArray \ptr -> action ptr `finally` releaseArrowArray ptr

withStruct :: (Storable a) => a -> (Ptr a -> IO b) -> IO b
withStruct initial action =
  alloca \ptr -> do
    poke ptr initial
    action ptr

withOwnedChunk :: DuckDBDataChunk -> (DuckDBDataChunk -> IO a) -> IO a
withOwnedChunk chunk = bracket (pure chunk) destroyChunk

withConvertedSchema :: DuckDBConnection -> Ptr ArrowSchema -> (DuckDBArrowConvertedSchema -> IO a) -> IO a
withConvertedSchema conn schemaPtr action =
  alloca \convertedPtr -> do
    poke convertedPtr (coerce nullPtr)
    err <- c_duckdb_schema_from_arrow conn (coerce schemaPtr) convertedPtr
    assertNoError err
    converted <- peek convertedPtr
    assertBool "converted schema pointer should not be null" (converted /= coerce nullPtr)
    bracket (pure converted) destroyArrowConvertedSchema action

destroyLogicalType :: DuckDBLogicalType -> IO ()
destroyLogicalType lt =
  alloca \ptr -> do
    poke ptr lt
    c_duckdb_destroy_logical_type ptr

destroyChunk :: DuckDBDataChunk -> IO ()
destroyChunk chunk =
  alloca \ptr -> do
    poke ptr chunk
    c_duckdb_destroy_data_chunk ptr

destroyArrowConvertedSchema :: DuckDBArrowConvertedSchema -> IO ()
destroyArrowConvertedSchema schema =
  alloca \ptr -> do
    poke ptr schema
    c_duckdb_destroy_arrow_converted_schema ptr

destroyErrorData :: DuckDBErrorData -> IO ()
destroyErrorData err =
  alloca \ptr -> do
    poke ptr err
    c_duckdb_destroy_error_data ptr

assertNoError :: DuckDBErrorData -> IO ()
assertNoError err =
  when (err /= coerce nullPtr) $ do
    msgPtr <- c_duckdb_error_data_message err
    msg <- peekCString (coerce msgPtr)
    destroyErrorData err
    assertFailure ("DuckDB reported error: " <> msg)

zeroArrowSchema :: ArrowSchema
zeroArrowSchema =
  ArrowSchema
    { format = coerce nullPtr
    , name = coerce nullPtr
    , metadata = coerce nullPtr
    , flags = 0
    , n_children = 0
    , children = nullPtr
    , dictionary = nullPtr
    , release = nullFunPtr
    , private_data = nullPtr
    }

zeroArrowArray :: ArrowArray
zeroArrowArray =
  ArrowArray
    { length = 0
    , null_count = 0
    , offset = 0
    , n_buffers = 0
    , n_children = 0
    , buffers = nullPtr
    , children = nullPtr
    , dictionary = nullPtr
    , release = nullFunPtr
    , private_data = nullPtr
    }