duckdb-ffi-2.0.0.0: test/ArrowInterfaceDeprecatedTests.hs
{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE TypeApplications #-}
{-# OPTIONS_GHC -Wno-deprecations #-}
module ArrowInterfaceDeprecatedTests (tests) where
import Control.Exception (bracket, finally)
import Control.Monad (unless, when, (>=>))
import Data.Coerce (coerce)
import Data.Int (Int32, Int64)
import Data.List (isInfixOf)
import Data.Proxy (Proxy (..))
import Database.DuckDB.FFI
import Foreign.C.ConstPtr (ConstPtr (..))
import Foreign.C.String (peekCString, peekCStringLen)
import Foreign.C.Types (CChar)
import Foreign.Marshal.Alloc (alloca)
import Foreign.Ptr (Ptr, nullFunPtr, nullPtr, plusPtr)
import Foreign.Storable (peek, peekElemOff, poke)
import GHC.Records (getField)
import HsBindgen.Runtime.HasCField (fromPtr)
import HsBindgen.Runtime.Support.FunPtr (fromFunPtr)
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=))
import Utils (releaseArrowArray, releaseArrowSchema, withConnection, withConstCString, withDatabase, withResult)
tests :: TestTree
tests =
testGroup
"Deprecated Arrow Interface"
[ queryArrowExposesSchemaAndArrays
, queryArrowReportsErrors
, preparedArrowSchemaMatchesStatement
, executePreparedArrowProducesRows
, resultArrowArrayMirrorsChunk
, arrowRowsChangedReflectsMutations
, arrowArrayScanRegistersView
, arrowStreamScanRegistersView
, arrowStructMovesPreserveOwnership
]
-- basic query ---------------------------------------------------------------
queryArrowExposesSchemaAndArrays :: TestTree
queryArrowExposesSchemaAndArrays =
testCase "query_arrow exposes schema metadata and arrays" $
withDatabase \db ->
withConnection db \conn ->
withSuccessfulArrow conn "SELECT 1::INTEGER AS id, 'duck'::VARCHAR AS label" \arrow -> do
columnCount <- c_duckdb_arrow_column_count arrow
columnCount @?= 2
rowCount <- c_duckdb_arrow_row_count arrow
rowCount @?= 1
rowsChanged <- c_duckdb_arrow_rows_changed arrow
rowsChanged @?= 0
errPtr <- c_duckdb_query_arrow_error arrow
when (errPtr /= coerce nullPtr) $ do
errMsg <- peekCString (coerce errPtr)
errMsg @?= ""
-- schema/array access validated in dedicated scan test
-- error handling ------------------------------------------------------------
queryArrowReportsErrors :: TestTree
queryArrowReportsErrors =
testCase "query_arrow surfaces execution errors" $
withDatabase \db ->
withConnection db \conn ->
withConstCString "SELECT * FROM missing_table" \querySql ->
alloca \arrowPtr -> do
poke arrowPtr (coerce nullPtr)
state <- c_duckdb_query_arrow conn querySql arrowPtr
state @?= DuckDBError
arrow <- peek arrowPtr
assertBool "arrow result should still be allocated on error" (arrow /= coerce nullPtr)
errPtr <- c_duckdb_query_arrow_error arrow
assertBool "error message should be present" (errPtr /= coerce nullPtr)
errMsg <- peekCString (coerce errPtr)
assertBool "error message should mention missing_table" ("missing_table" `isInfixOf` errMsg)
c_duckdb_destroy_arrow arrowPtr
-- prepared statements -------------------------------------------------------
preparedArrowSchemaMatchesStatement :: TestTree
preparedArrowSchemaMatchesStatement =
testCase "prepared_arrow_schema describes statement parameters" $
withDatabase \db ->
withConnection db \conn ->
withPrepared conn "SELECT ?::INTEGER, ?::VARCHAR" \stmt ->
alloca \schemaStorage -> do
poke schemaStorage zeroArrowSchema
alloca \schemaOut -> do
poke schemaOut (coerce schemaStorage)
c_duckdb_prepared_arrow_schema stmt schemaOut >>= (@?= DuckDBSuccess)
schema <- peek schemaStorage
(getField @"n_children") schema @?= 2
children <- mapM (peekElemOff ((getField @"children") schema) >=> peek) [0, 1]
mapM ((peekCString . coerce) . (getField @"name")) children >>= (@?= ["0", "1"])
mapM ((peekCString . coerce) . (getField @"format")) children >>= (@?= ["n", "n"])
releaseArrowSchema schemaStorage
executePreparedArrowProducesRows :: TestTree
executePreparedArrowProducesRows =
testCase "execute_prepared_arrow materialises a result set" $
withDatabase \db ->
withConnection db \conn ->
withPrepared conn "SELECT ?::INTEGER + 5 AS computed" \stmt -> do
c_duckdb_bind_int32 stmt 1 (5 :: Int32) >>= (@?= DuckDBSuccess)
withPreparedArrow stmt \arrow -> do
rowCount <- c_duckdb_arrow_row_count arrow
rowCount @?= 1
colCount <- c_duckdb_arrow_column_count arrow
colCount @?= 1
-- result conversion ---------------------------------------------------------
resultArrowArrayMirrorsChunk :: TestTree
resultArrowArrayMirrorsChunk =
-- NOTE: duckdb_result_arrow_array writes release callbacks into the
-- provided ArrowArray storage. Always zero-initialise the struct
-- (see zeroArrowArray) and honour the release function before the
-- stack memory goes out of scope.
testCase "result_arrow_array converts materialised chunks to Arrow arrays" $
withDatabase \db ->
withConnection db \conn -> do
execStatement conn "CREATE TABLE arrow_chunks(id BIGINT, label VARCHAR);"
execStatement conn "INSERT INTO arrow_chunks VALUES (10, 'ten'), (11, 'eleven');"
withResult conn "SELECT id, label FROM arrow_chunks ORDER BY id" \resPtr -> do
chunk <- peek resPtr >>= \rawValue -> c_duckdb_result_get_chunk rawValue 0
assertBool "fetch_chunk returned a null chunk" (chunk /= coerce nullPtr)
chunkSize <- c_duckdb_data_chunk_get_size chunk
chunkCols <- c_duckdb_data_chunk_get_column_count chunk
alloca \arrowArrayPtr -> do
poke arrowArrayPtr zeroArrowArray
let duckArray :: DuckDBArrowArray
duckArray = coerce arrowArrayPtr
alloca \arrayOut -> do
poke arrayOut duckArray
peek resPtr >>= \rawValue -> c_duckdb_result_arrow_array rawValue chunk arrayOut
array <- peek arrowArrayPtr
(getField @"length") array @?= fromIntegral chunkSize
(getField @"n_children") array @?= fromIntegral chunkCols
validateChunkChildren array
releaseArrowArray arrowArrayPtr
destroyChunk chunk
-- rows changed --------------------------------------------------------------
arrowRowsChangedReflectsMutations :: TestTree
arrowRowsChangedReflectsMutations =
testCase "arrow_rows_changed reports mutation counts" $
withDatabase \db ->
withConnection db \conn -> do
execStatement conn "CREATE TABLE arrow_changes(val INTEGER);"
withSuccessfulArrow conn "INSERT INTO arrow_changes VALUES (1), (2), (3)" \arrow -> do
rowCount <- c_duckdb_arrow_row_count arrow
assertBool "modification result should not report negative rows" (rowCount >= 0)
changed <- c_duckdb_arrow_rows_changed arrow
assertBool "rows_changed should report positive count" (changed > 0)
-- arrow scans ----------------------------------------------------------------
arrowArrayScanRegistersView :: TestTree
arrowArrayScanRegistersView =
-- NOTE: Both the schema and array buffers must be initialised to
-- zeroed Arrow structures before calling the query helpers. DuckDB
-- fills in release callbacks and expects us to invoke
-- duckdb_destroy_arrow_stream on the out stream.
testCase "arrow_array_scan registers a view and yields a release stream" $
withDatabase \db ->
withConnection db \conn -> do
execStatement conn "CREATE TABLE arrow_scan_source(i BIGINT, label VARCHAR);"
execStatement conn "INSERT INTO arrow_scan_source VALUES (5, 'five'), (6, 'six');"
withSuccessfulArrow conn "SELECT i, label FROM arrow_scan_source ORDER BY i" \arrow -> do
alloca \schemaStorage -> do
poke schemaStorage zeroArrowSchema
let schemaHandle = coerce schemaStorage :: DuckDBArrowSchema
alloca \schemaOut -> do
poke schemaOut schemaHandle
schemaState <- c_duckdb_query_arrow_schema arrow schemaOut
schemaState @?= DuckDBSuccess
alloca \arrayStorage -> do
poke arrayStorage zeroArrowArray
let arrayHandle = coerce arrayStorage :: DuckDBArrowArray
alloca \arrayOut -> do
poke arrayOut arrayHandle
arrayState <- c_duckdb_query_arrow_array arrow arrayOut
arrayState @?= DuckDBSuccess
withConstCString "arrow_array_view" \viewName ->
alloca \streamOut -> do
poke streamOut (coerce nullPtr)
scanState <- c_duckdb_arrow_array_scan conn viewName schemaHandle arrayHandle streamOut
scanState @?= DuckDBSuccess
streamWrapper <- peek streamOut
assertBool "arrow_array_scan returned a null stream handle" (streamWrapper /= coerce nullPtr)
withResult conn "SELECT COUNT(*) FROM arrow_array_view" \resPtr -> do
count <- c_duckdb_value_int64 resPtr 0 0
count @?= 2
c_duckdb_destroy_arrow_stream streamOut
releaseArrowArray arrayStorage
releaseArrowSchema schemaStorage
arrowStreamScanRegistersView :: TestTree
arrowStreamScanRegistersView =
-- NOTE: duckdb_arrow_array_scan returns DuckDBError if the target
-- view name already exists (even if it is a table). Use a fresh
-- view name that will be dropped implicitly when the connection
-- closes.
testCase "arrow_scan registers a view from an Arrow stream" $
withDatabase \db ->
withConnection db \conn -> do
execStatement conn "CREATE TABLE arrow_stream_source(i BIGINT, label VARCHAR);"
execStatement conn "INSERT INTO arrow_stream_source VALUES (7, 'seven'), (8, 'eight');"
withSuccessfulArrow conn "SELECT i, label FROM arrow_stream_source ORDER BY i" \arrow -> do
alloca \schemaStorage ->
finally
( do
poke schemaStorage zeroArrowSchema
let schemaHandle = coerce schemaStorage :: DuckDBArrowSchema
alloca \schemaOut -> do
poke schemaOut schemaHandle
schemaState <- c_duckdb_query_arrow_schema arrow schemaOut
schemaState @?= DuckDBSuccess
alloca \arrayStorage ->
finally
( do
poke arrayStorage zeroArrowArray
let arrayHandle = coerce arrayStorage :: DuckDBArrowArray
alloca \arrayOut -> do
poke arrayOut arrayHandle
arrayState <- c_duckdb_query_arrow_array arrow arrayOut
arrayState @?= DuckDBSuccess
withConstCString "arrow_stream_array_view" \sourceView ->
alloca \streamOut -> do
poke streamOut (coerce nullPtr)
arrayScanState <- c_duckdb_arrow_array_scan conn sourceView schemaHandle arrayHandle streamOut
arrayScanState @?= DuckDBSuccess
streamHandle <- peek streamOut
assertBool "arrow_array_scan returned a null stream" (streamHandle /= coerce nullPtr)
withConstCString "arrow_stream_view" \streamView -> do
streamScanState <- c_duckdb_arrow_scan conn streamView streamHandle
streamScanState @?= DuckDBSuccess
withResult conn "SELECT COUNT(*) FROM arrow_stream_view" \resPtr -> do
count <- c_duckdb_value_int64 resPtr 0 0
count @?= 2
c_duckdb_destroy_arrow_stream streamOut
)
(releaseArrowArray arrayStorage)
)
(releaseArrowSchema schemaStorage)
arrowStructMovesPreserveOwnership :: TestTree
arrowStructMovesPreserveOwnership =
testCase "Arrow structs move real DuckDB buffers without releasing them" $ do
withDatabase \db ->
withConnection db \conn ->
withSuccessfulArrow conn "SELECT 42::BIGINT AS id, 'duck' AS label" \arrow ->
alloca \schemaStorage -> do
poke schemaStorage zeroArrowSchema
let schemaHandle = coerce schemaStorage :: DuckDBArrowSchema
alloca \schemaOut -> do
poke schemaOut schemaHandle
c_duckdb_query_arrow_schema arrow schemaOut >>= (@?= DuckDBSuccess)
alloca \arrayStorage -> do
poke arrayStorage zeroArrowArray
let arrayHandle = coerce arrayStorage :: DuckDBArrowArray
alloca \arrayOut -> do
poke arrayOut arrayHandle
c_duckdb_query_arrow_array arrow arrayOut >>= (@?= DuckDBSuccess)
withConstCString "arrow_helper_view" \viewName ->
alloca \streamOut -> do
poke streamOut (coerce nullPtr)
c_duckdb_arrow_array_scan conn viewName schemaHandle arrayHandle streamOut >>= (@?= DuckDBSuccess)
stream <- peek streamOut
original <- peek (coerce stream :: Ptr ArrowArrayStream)
alloca \movedStream -> do
poke movedStream original
poke (fromPtr (Proxy @"release") ((coerce stream :: Ptr ArrowArrayStream) :: Ptr ArrowArrayStream)) nullFunPtr
alloca \movedSchema -> do
poke movedSchema zeroArrowSchema
fromFunPtr ((getField @"get_schema") original) movedStream movedSchema >>= (@?= 0)
schema <- peek movedSchema
(peekCString . coerce) ((getField @"format") schema) >>= (@?= "+s")
releaseArrowSchema movedSchema
alloca \movedArray -> do
poke movedArray zeroArrowArray
fromFunPtr ((getField @"get_next") original) movedStream movedArray >>= (@?= 0)
batch <- peek movedArray
(getField @"length") batch @?= 1
releaseArrowArray movedArray
fromFunPtr ((getField @"get_next") original) movedStream movedArray >>= (@?= 0)
exhausted <- peek movedArray
(getField @"release") exhausted @?= nullFunPtr
fromFunPtr ((getField @"release") original) movedStream
moved <- peek movedStream
(getField @"release") moved @?= nullFunPtr
c_duckdb_destroy_arrow_stream streamOut
peek streamOut >>= (@?= coerce nullPtr)
originalArray <- peek arrayStorage
alloca \movedArray -> do
poke movedArray originalArray
poke (fromPtr (Proxy @"release") (arrayStorage :: Ptr ArrowArray)) nullFunPtr
moved <- peek movedArray
(getField @"length") moved @?= 1
child <- peekElemOff ((getField @"children") moved) 0 >>= peek
buffer <- peekElemOff ((getField @"buffers") child) 1
peek (coerce buffer :: Ptr Int64) >>= (@?= 42)
releaseArrowArray movedArray
poke (fromPtr (Proxy @"release") (arrayStorage :: Ptr ArrowArray)) nullFunPtr
originalSchema <- peek schemaStorage
alloca \movedSchema -> do
poke movedSchema originalSchema
poke (fromPtr (Proxy @"release") (schemaStorage :: Ptr ArrowSchema)) nullFunPtr
moved <- peek movedSchema
(peekCString . coerce) ((getField @"format") moved) >>= (@?= "+s")
child <- peekElemOff ((getField @"children") moved) 0 >>= peek
(peekCString . coerce) ((getField @"name") child) >>= (@?= "id")
releaseArrowSchema movedSchema
poke (fromPtr (Proxy @"release") (schemaStorage :: Ptr ArrowSchema)) nullFunPtr
zeroArrowArray :: ArrowArray
zeroArrowArray =
ArrowArray
{ length = 0
, null_count = 0
, offset = 0
, n_buffers = 0
, n_children = 0
, buffers = nullPtr
, children = nullPtr
, dictionary = nullPtr
, release = nullFunPtr
, private_data = nullPtr
}
zeroArrowSchema :: ArrowSchema
zeroArrowSchema =
ArrowSchema
{ format = coerce nullPtr
, name = coerce nullPtr
, metadata = coerce nullPtr
, flags = 0
, n_children = 0
, children = nullPtr
, dictionary = nullPtr
, release = nullFunPtr
, private_data = nullPtr
}
validateChunkChildren :: ArrowArray -> IO ()
validateChunkChildren array = do
let expectedIds = [10, 11] :: [Int64]
expectedLabels = ["ten", "eleven"]
childCount = fromIntegral ((getField @"n_children") array) :: Int
rowCount = fromIntegral ((getField @"length") array) :: Int
childCount @?= 2
rowCount @?= length expectedIds
let childrenPtr = (getField @"children") array
assertBool "Arrow array did not expose child arrays" (childrenPtr /= nullPtr)
let ensurePtr name ptrPred =
unless ptrPred $
assertFailure ("Arrow array " ++ name ++ " pointer is null")
intChildPtr <- peekElemOff childrenPtr 0
ensurePtr "integer child" (intChildPtr /= nullPtr)
intChild <- peek intChildPtr
(getField @"null_count") intChild @?= 0
let intBufferCount = fromIntegral ((getField @"n_buffers") intChild) :: Int
assertBool "Integer child did not expose the expected buffers" (intBufferCount >= 2)
let intBuffers = (getField @"buffers") intChild
ensurePtr "integer buffers" (intBuffers /= nullPtr)
valueBufferRaw <- peekElemOff intBuffers 1
ensurePtr "integer value buffer" (valueBufferRaw /= coerce nullPtr)
let valueBuffer = coerce valueBufferRaw :: Ptr Int64
values <- mapM (peekElemOff valueBuffer) [0 .. rowCount - 1]
values @?= expectedIds
strChildPtr <- peekElemOff childrenPtr 1
ensurePtr "varchar child" (strChildPtr /= nullPtr)
strChild <- peek strChildPtr
(getField @"null_count") strChild @?= 0
let strBufferCount = fromIntegral ((getField @"n_buffers") strChild) :: Int
assertBool "Varchar child did not expose the expected buffers" (strBufferCount >= 3)
let strBuffers = (getField @"buffers") strChild
ensurePtr "varchar buffers" (strBuffers /= nullPtr)
offsetsRaw <- peekElemOff strBuffers 1
dataRaw <- peekElemOff strBuffers 2
ensurePtr "varchar offsets buffer" (offsetsRaw /= coerce nullPtr)
ensurePtr "varchar data buffer" (dataRaw /= coerce nullPtr)
let offsetsPtr = coerce offsetsRaw :: Ptr Int32
dataPtr = coerce dataRaw :: Ptr CChar
labels <-
mapM
( \idx -> do
start <- fromIntegral <$> peekElemOff offsetsPtr idx
end <- fromIntegral <$> peekElemOff offsetsPtr (idx + 1)
peekCStringLen (dataPtr `plusPtr` start, end - start)
)
[0 .. rowCount - 1]
labels @?= expectedLabels
withSuccessfulArrow :: DuckDBConnection -> String -> (DuckDBArrow -> IO a) -> IO a
withSuccessfulArrow conn sql action =
withConstCString sql \sqlPtr ->
alloca \arrowPtr ->
bracket
( do
poke arrowPtr (coerce nullPtr)
state <- c_duckdb_query_arrow conn sqlPtr arrowPtr
state @?= DuckDBSuccess
arrow <- peek arrowPtr
assertBool "duckdb_query_arrow returned null result" (arrow /= coerce nullPtr)
pure arrow
)
(\_ -> c_duckdb_destroy_arrow arrowPtr)
action
withPrepared :: DuckDBConnection -> String -> (DuckDBPreparedStatement -> IO a) -> IO a
withPrepared conn sql action =
withConstCString sql \sqlPtr ->
alloca \stmtPtr ->
bracket
( do
state <- c_duckdb_prepare conn sqlPtr stmtPtr
state @?= DuckDBSuccess
stmt <- peek stmtPtr
assertBool "prepare should produce a statement" (stmt /= coerce nullPtr)
pure stmt
)
(\_ -> c_duckdb_destroy_prepare stmtPtr)
action
withPreparedArrow :: DuckDBPreparedStatement -> (DuckDBArrow -> IO a) -> IO a
withPreparedArrow stmt action =
alloca \arrowPtr ->
bracket
( do
poke arrowPtr (coerce nullPtr)
state <- c_duckdb_execute_prepared_arrow stmt arrowPtr
state @?= DuckDBSuccess
arrow <- peek arrowPtr
assertBool "execute_prepared_arrow returned null result" (arrow /= coerce nullPtr)
pure arrow
)
(\_ -> c_duckdb_destroy_arrow arrowPtr)
action
destroyChunk :: DuckDBDataChunk -> IO ()
destroyChunk chunk =
alloca \ptr -> poke ptr chunk >> c_duckdb_destroy_data_chunk ptr
execStatement :: DuckDBConnection -> String -> IO ()
execStatement conn sql =
withConstCString sql \sqlPtr ->
alloca \resPtr -> do
st <- c_duckdb_query conn sqlPtr resPtr
st @?= DuckDBSuccess
c_duckdb_destroy_result resPtr