duckdb-ffi-2.0.0.0: test/TableFunctionsTest.hs
{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE TypeApplications #-}
{-# OPTIONS_GHC -Wno-deprecations #-}
module TableFunctionsTest (tests) where
import Control.Concurrent (runInBoundThread)
import Control.Exception (bracket)
import Control.Monad (forM_, when)
import Data.Char (isAsciiUpper)
import Data.Coerce (coerce)
import Data.IORef (IORef, modifyIORef', newIORef, readIORef, writeIORef)
import Data.Int (Int64)
import Data.List (isInfixOf)
import Data.Maybe (fromMaybe)
import Data.Proxy (Proxy (..))
import Data.Void (Void)
import Database.DuckDB.FFI
import Foreign.C.ConstPtr (ConstPtr (..))
import Foreign.C.String (peekCString)
import Foreign.C.Types (CBool (..), CChar)
import Foreign.Marshal.Alloc (alloca, free, mallocBytes)
import Foreign.Ptr (Ptr, freeHaskellFunPtr, nullPtr)
import Foreign.Storable (peek, peekElemOff, poke, pokeElemOff)
import HsBindgen.Runtime.Marshal (staticSizeOf)
import HsBindgen.Runtime.Support.FunPtr (toFunPtr)
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit (assertBool, assertFailure, testCase, (@?=))
import Utils (withConnection, withConstCString, withDatabase, withLogicalType, withResultCString, withValue)
tests :: TestTree
tests =
testGroup
"Table Functions"
[tableFunctionLifecycle]
tableFunctionLifecycle :: TestTree
tableFunctionLifecycle =
testCase "table function lifecycle covers core callbacks" $
runInBoundThread do
withDatabase \db ->
withLogicalType (c_duckdb_create_logical_type (DuckDBTypeAlias DuckDBTypeBigInt)) \bigint ->
withHarness bigint \harness@TableHarness{} -> do
(bindError, execError, initError) <-
withConnection db \conn ->
withTableFunction \tableFun -> do
configureTableFunction tableFun harness
c_duckdb_register_table_function conn tableFun >>= (@?= DuckDBSuccess)
withConstCString "SELECT value, double_value FROM haskell_numbers(5, start => 2)" \sql ->
withResultCString conn sql \resPtr -> do
c_duckdb_column_count resPtr >>= (@?= 2)
c_duckdb_row_count resPtr >>= (@?= 5)
forM_ [0 .. 4] \idx -> do
c_duckdb_value_int64 resPtr 0 idx >>= (@?= 2 + fromIntegral idx)
c_duckdb_value_int64 resPtr 1 idx >>= (@?= 2 * (2 + fromIntegral idx))
withConstCString "SELECT value FROM haskell_numbers(3)" \sql ->
withResultCString conn sql \resPtr -> do
c_duckdb_column_count resPtr >>= (@?= 1)
forM_ (zip [0 ..] [0 .. 2]) \(idx, expected) ->
c_duckdb_value_int64 resPtr 0 idx >>= (@?= expected)
bindError <-
withConstCString "SELECT * FROM haskell_numbers(0)" \sql ->
alloca \resPtr -> do
state <- c_duckdb_query conn sql resPtr
errPtr <- c_duckdb_result_error resPtr
msg <- peekCStringMaybe errPtr
c_duckdb_destroy_result resPtr
pure (state, msg)
execError <-
withConstCString "SELECT * FROM haskell_numbers(5, fail_at => 3)" \sql ->
alloca \resPtr -> do
state <- c_duckdb_query conn sql resPtr
errPtr <- c_duckdb_result_error resPtr
msg <- peekCStringMaybe errPtr
c_duckdb_destroy_result resPtr
pure (state, msg)
initError <-
withConstCString "SELECT * FROM haskell_numbers(2, force_init_error => 1)" \sql ->
alloca \resPtr -> do
state <- c_duckdb_query conn sql resPtr
errPtr <- c_duckdb_result_error resPtr
msg <- peekCStringMaybe errPtr
c_duckdb_destroy_result resPtr
pure (state, msg)
pure (bindError, execError, initError)
stats <- snapshotStats harness
hsBindFreed stats @?= 3
hsInitFreed stats @?= 3
hsLocalFreed stats @?= 3
assertBool "client context seen" (hsClientContextSeen stats)
hsCounts stats @?= [5, 3, 0, 5, 2]
hsProjectionHistory stats
@?= [ ProjectionSnapshot 2 [0, 1]
, ProjectionSnapshot 1 [0]
, ProjectionSnapshot 2 [0, 1]
]
fst bindError @?= DuckDBError
assertBool "bind error mentions positivity" ("positive" `contains` snd bindError)
fst execError @?= DuckDBError
assertBool "execution error mentions table function" ("execution" `contains` snd execError)
fst initError @?= DuckDBError
assertBool "init error mentions initialization" ("init" `contains` snd initError)
-- Harness --------------------------------------------------------------------
data ProjectionSnapshot = ProjectionSnapshot
{ psCount :: !Int
, psIndices :: ![Int]
}
deriving (Eq, Show)
data ExecutionMode
= ModeUnset
| ModeNormal
| ModeInitError
| ModeRejected
deriving (Eq, Show)
data TableHarness = TableHarness
{ thColumnType :: !DuckDBLogicalType
, thExtraBuffer :: !(Ptr Void)
, thExtraCount :: !(IORef Int)
, thBindCount :: !(IORef Int)
, thInitCount :: !(IORef Int)
, thLocalCount :: !(IORef Int)
, thBindPtr :: !(IORef (Maybe (Ptr Void)))
, thInitPtr :: !(IORef (Maybe (Ptr Void)))
, thLocalPtr :: !(IORef (Maybe (Ptr Void)))
, thProjectionCurrent :: !(IORef (Maybe ProjectionSnapshot))
, thProjectionHistory :: !(IORef [ProjectionSnapshot])
, thClientContextSeen :: !(IORef Bool)
, thModeRef :: !(IORef ExecutionMode)
, thCountHistory :: !(IORef [Int64])
, thCallbacks :: !TableCallbacks
, thDeletes :: !DeleteCallbacks
}
data TableCallbacks = TableCallbacks
{ tcBind :: !DuckDBTableFunctionBindFun
, tcInit :: !DuckDBTableFunctionInitFun
, tcLocalInit :: !DuckDBTableFunctionInitFun
, tcExecute :: !DuckDBTableFunctionFun
}
data DeleteCallbacks = DeleteCallbacks
{ dcExtra :: !DuckDBDeleteCallback
, dcBind :: !DuckDBDeleteCallback
, dcInit :: !DuckDBDeleteCallback
, dcLocal :: !DuckDBDeleteCallback
}
data HarnessStats = HarnessStats
{ hsExtraFreed :: !Int
, hsBindFreed :: !Int
, hsInitFreed :: !Int
, hsLocalFreed :: !Int
, hsClientContextSeen :: !Bool
, hsProjectionHistory :: ![ProjectionSnapshot]
, hsCounts :: ![Int64]
}
deriving (Eq, Show)
withHarness :: DuckDBLogicalType -> (TableHarness -> IO a) -> IO a
withHarness columnType action = do
rawExtraBuffer <- mallocBytes sizeOfInt64
let extraBuffer = coerce rawExtraBuffer :: Ptr Void
poke (coerce extraBuffer :: Ptr Int64) extraInfoSentinel
extraCount <- newIORef 0
bindCount <- newIORef 0
initCount <- newIORef 0
localCount <- newIORef 0
bindPtrRef <- newIORef Nothing
initPtrRef <- newIORef Nothing
localPtrRef <- newIORef Nothing
projectionCurrent <- newIORef Nothing
projectionHistory <- newIORef []
clientContextSeen <- newIORef False
modeRef <- newIORef ModeUnset
countHistory <- newIORef []
let takeInt64 :: Ptr Void -> Int -> IO Int64
takeInt64 ptr = peekElemOff (coerce ptr :: Ptr Int64)
putInt64 :: Ptr Void -> Int -> Int64 -> IO ()
putInt64 ptr = pokeElemOff (coerce ptr :: Ptr Int64)
extraDelete <-
mkDeleteCallback \ptr -> do
modifyIORef' extraCount (+ 1)
when (ptr /= nullPtr) (free ptr)
bindDelete <-
mkDeleteCallback \ptr -> do
modifyIORef' bindCount (+ 1)
writeIORef bindPtrRef Nothing
when (ptr /= nullPtr) (free ptr)
initDelete <-
mkDeleteCallback \ptr -> do
modifyIORef' initCount (+ 1)
writeIORef initPtrRef Nothing
when (ptr /= nullPtr) (free ptr)
localDelete <-
mkDeleteCallback \ptr -> do
modifyIORef' localCount (+ 1)
writeIORef localPtrRef Nothing
when (ptr /= nullPtr) (free ptr)
let resetLifecycle = do
writeIORef projectionCurrent Nothing
writeIORef modeRef ModeUnset
writeIORef bindPtrRef Nothing
writeIORef initPtrRef Nothing
writeIORef localPtrRef Nothing
recordProjection snapshot = do
writeIORef projectionCurrent (Just snapshot)
modifyIORef' projectionHistory (<> [snapshot])
bindCallback info = do
resetLifecycle
writeIORef modeRef ModeRejected
extraPtr <- c_duckdb_bind_get_extra_info info
extraPtr @?= extraBuffer
sentinel <- peek (coerce extraPtr :: Ptr Int64)
sentinel @?= extraInfoSentinel
alloca \ctxPtr -> do
poke ctxPtr (coerce nullPtr)
c_duckdb_table_function_get_client_context info ctxPtr
ctx <- peek ctxPtr
assertBool "client context should not be null" (ctx /= coerce nullPtr)
cid <- c_duckdb_client_context_get_connection_id ctx
assertBool "connection id should be non-negative" (cid >= 0)
c_duckdb_destroy_client_context ctxPtr
writeIORef clientContextSeen True
paramCount <- c_duckdb_bind_get_parameter_count info
paramCount @?= 1
withValue (c_duckdb_bind_get_parameter info 0) \value -> do
countValue <- c_duckdb_get_int64 value
modifyIORef' countHistory (<> [countValue])
if countValue <= 0
then withConstCString "count must be positive" $ \msg -> c_duckdb_bind_set_error info msg
else do
startValue <-
withConstCString "start" \name ->
fmap (fromMaybe 0) (withOptionalValue (c_duckdb_bind_get_named_parameter info name) c_duckdb_get_int64)
failAtValue <-
withConstCString "fail_at" \name ->
fmap (fromMaybe (-1)) (withOptionalValue (c_duckdb_bind_get_named_parameter info name) c_duckdb_get_int64)
initErrValue <-
withConstCString "force_init_error" \name ->
fmap (fromMaybe 0) (withOptionalValue (c_duckdb_bind_get_named_parameter info name) c_duckdb_get_int64)
absentValue <-
withConstCString "missing" \name ->
withOptionalValue (c_duckdb_bind_get_named_parameter info name) c_duckdb_get_int64
absentValue @?= Nothing
withConstCString "value" \col -> c_duckdb_bind_add_result_column info col columnType
withConstCString "double_value" \col -> c_duckdb_bind_add_result_column info col columnType
c_duckdb_bind_set_cardinality info (fromIntegral countValue) (toCBool True)
if initErrValue /= 0
then do
writeIORef modeRef ModeInitError
writeIORef bindPtrRef Nothing
else do
cfgPtr <- mallocBytes (3 * sizeOfInt64)
putInt64 cfgPtr 0 startValue
putInt64 cfgPtr 1 countValue
putInt64 cfgPtr 2 failAtValue
c_duckdb_bind_set_bind_data info (coerce cfgPtr) bindDelete
writeIORef bindPtrRef (Just (coerce cfgPtr))
writeIORef modeRef ModeNormal
initCallback info = do
extraPtr <- c_duckdb_init_get_extra_info info
extraPtr @?= extraBuffer
mode <- readIORef modeRef
case mode of
ModeInitError -> do
bindRaw <- c_duckdb_init_get_bind_data info
bindRaw @?= nullPtr
withConstCString "init rejected missing bind data" $ \msg -> c_duckdb_init_set_error info msg
ModeNormal -> do
bindRaw <- c_duckdb_init_get_bind_data info
stored <- readIORef bindPtrRef
case stored of
Nothing -> assertFailure "bind data expected during init"
Just expected -> bindRaw @?= expected
startValue <- takeInt64 bindRaw 0
countValue <- takeInt64 bindRaw 1
failAtValue <- takeInt64 bindRaw 2
columnCount <- fromIntegral <$> c_duckdb_init_get_column_count info
indices <-
mapM
(fmap (fromIntegral :: DuckDBIdx -> Int) . c_duckdb_init_get_column_index info . fromIntegral)
[0 .. columnCount - 1]
recordProjection ProjectionSnapshot{psCount = columnCount, psIndices = indices}
statePtr <- mallocBytes (3 * sizeOfInt64)
putInt64 statePtr 0 startValue
putInt64 statePtr 1 (startValue + countValue)
putInt64 statePtr 2 failAtValue
c_duckdb_init_set_init_data info (coerce statePtr) initDelete
writeIORef initPtrRef (Just (coerce statePtr))
c_duckdb_init_set_max_threads info 1
_ -> pure ()
localInitCallback info = do
extraPtr <- c_duckdb_init_get_extra_info info
extraPtr @?= extraBuffer
mode <- readIORef modeRef
case mode of
ModeNormal -> do
bindRaw <- c_duckdb_init_get_bind_data info
stored <- readIORef bindPtrRef
case stored of
Nothing -> assertFailure "bind data should be present during local init"
Just expected -> bindRaw @?= expected
localPtr <- mallocBytes sizeOfInt64
poke (coerce localPtr :: Ptr Int64) localInitSentinel
c_duckdb_init_set_init_data info (coerce localPtr) localDelete
writeIORef localPtrRef (Just (coerce localPtr))
ModeInitError -> withConstCString "local init skipped after init error" $ \msg -> c_duckdb_init_set_error info msg
_ -> pure ()
executeCallback info chunk = do
extraPtr <- c_duckdb_function_get_extra_info info
extraPtr @?= extraBuffer
mode <- readIORef modeRef
case mode of
ModeNormal -> do
bindRaw <- c_duckdb_function_get_bind_data info
storedBind <- readIORef bindPtrRef
case storedBind of
Nothing -> assertFailure "bind data should be recorded before execution"
Just expected -> bindRaw @?= expected
initRaw <- c_duckdb_function_get_init_data info
storedInit <- readIORef initPtrRef
case storedInit of
Nothing -> assertFailure "init data should exist during execution"
Just expected -> initRaw @?= expected
localRaw <- c_duckdb_function_get_local_init_data info
storedLocal <- readIORef localPtrRef
case storedLocal of
Nothing -> assertFailure "local init data should exist during execution"
Just expected -> localRaw @?= expected
sentinel <- peek (coerce localRaw :: Ptr Int64)
sentinel @?= localInitSentinel
snapshot <- readIORef projectionCurrent
ProjectionSnapshot{..} <-
case snapshot of
Nothing -> do
_ <- assertFailure "projection snapshot missing during execution"
pure (ProjectionSnapshot 0 [])
Just snap -> pure snap
assertBool "projection slots match indices" (length psIndices == psCount)
nextVal <- takeInt64 initRaw 0
endVal <- takeInt64 initRaw 1
failAtValue <- takeInt64 initRaw 2
if nextVal >= endVal
then c_duckdb_data_chunk_set_size chunk 0
else do
let remaining = endVal - nextVal
batch = min remaining 1024
chunkEnd = nextVal + batch
if failAtValue >= nextVal && failAtValue < chunkEnd && failAtValue >= 0
then do
withConstCString "execution aborted by table function" $ \msg -> c_duckdb_function_set_error info msg
c_duckdb_data_chunk_set_size chunk 0
putInt64 initRaw 0 chunkEnd
else do
forM_ (zip [0 :: Integer ..] psIndices) \(slot, schemaIdx) -> do
vec <- c_duckdb_data_chunk_get_vector chunk (fromIntegral slot)
dataPtr <- vectorDataPtr vec
forM_ [0 :: Integer .. fromIntegral batch - 1] \offset -> do
let base = nextVal + fromIntegral offset
val =
case schemaIdx of
0 -> base
1 -> 2 * base
_ -> 0
pokeElemOff dataPtr (fromIntegral offset) val
c_duckdb_data_chunk_set_size chunk (fromIntegral batch)
putInt64 initRaw 0 chunkEnd
ModeInitError -> do
withConstCString "execution blocked by init error" $ \msg -> c_duckdb_function_set_error info msg
c_duckdb_data_chunk_set_size chunk 0
ModeRejected -> do
withConstCString "execution not expected after bind error" $ \msg -> c_duckdb_function_set_error info msg
c_duckdb_data_chunk_set_size chunk 0
ModeUnset -> do
withConstCString "execution invoked without prior bind" $ \msg -> c_duckdb_function_set_error info msg
c_duckdb_data_chunk_set_size chunk 0
bindFun <- mkBindFun bindCallback
initFun <- mkInitFun initCallback
localInitFun <- mkInitFun localInitCallback
execFun <- mkFunctionFun executeCallback
let callbacks =
TableCallbacks
{ tcBind = bindFun
, tcInit = initFun
, tcLocalInit = localInitFun
, tcExecute = execFun
}
deletes =
DeleteCallbacks
{ dcExtra = extraDelete
, dcBind = bindDelete
, dcInit = initDelete
, dcLocal = localDelete
}
let harness =
TableHarness
{ thColumnType = columnType
, thExtraBuffer = extraBuffer
, thExtraCount = extraCount
, thBindCount = bindCount
, thInitCount = initCount
, thLocalCount = localCount
, thBindPtr = bindPtrRef
, thInitPtr = initPtrRef
, thLocalPtr = localPtrRef
, thProjectionCurrent = projectionCurrent
, thProjectionHistory = projectionHistory
, thClientContextSeen = clientContextSeen
, thModeRef = modeRef
, thCountHistory = countHistory
, thCallbacks = callbacks
, thDeletes = deletes
}
result <- action harness
freeHaskellFunPtr (coerce bindFun)
freeHaskellFunPtr (coerce initFun)
freeHaskellFunPtr (coerce localInitFun)
freeHaskellFunPtr (coerce execFun)
-- DuckDB may still invoke delete callbacks while the table function tears down.
-- Rely on process teardown to reclaim them to avoid racing with finalizers.
pure result
withTableFunction :: (DuckDBTableFunction -> IO a) -> IO a
withTableFunction = bracket c_duckdb_create_table_function destroy
where
destroy fun = alloca \ptr -> poke ptr fun >> c_duckdb_destroy_table_function ptr
configureTableFunction :: DuckDBTableFunction -> TableHarness -> IO ()
configureTableFunction fun TableHarness{..} = do
withConstCString "haskell_numbers" $ \name -> c_duckdb_table_function_set_name fun name
c_duckdb_table_function_add_parameter fun thColumnType
withConstCString "start" \n -> c_duckdb_table_function_add_named_parameter fun n thColumnType
withConstCString "fail_at" \n -> c_duckdb_table_function_add_named_parameter fun n thColumnType
withConstCString "force_init_error" \n -> c_duckdb_table_function_add_named_parameter fun n thColumnType
-- NOTE: DuckDB expects the extra-info pointer to be populated alongside the other callbacks;
-- leaving it unset would skip the destructor and break the lifetime assertions below.
c_duckdb_table_function_set_extra_info fun thExtraBuffer (dcExtra thDeletes)
c_duckdb_table_function_set_bind fun (tcBind thCallbacks)
c_duckdb_table_function_set_init fun (tcInit thCallbacks)
c_duckdb_table_function_set_local_init fun (tcLocalInit thCallbacks)
c_duckdb_table_function_set_function fun (tcExecute thCallbacks)
c_duckdb_table_function_supports_projection_pushdown fun (toCBool True)
snapshotStats :: TableHarness -> IO HarnessStats
snapshotStats TableHarness{..} = do
hsExtraFreed <- readIORef thExtraCount
hsBindFreed <- readIORef thBindCount
hsInitFreed <- readIORef thInitCount
hsLocalFreed <- readIORef thLocalCount
hsClientContextSeen <- readIORef thClientContextSeen
history <- readIORef thProjectionHistory
hsCounts <- readIORef thCountHistory
pure
HarnessStats
{ hsProjectionHistory = history
, ..
}
-- Helpers --------------------------------------------------------------------
extraInfoSentinel :: Int64
extraInfoSentinel = 0xdeadbeef42
localInitSentinel :: Int64
localInitSentinel = 0x105105105
sizeOfInt64 :: Int
sizeOfInt64 = staticSizeOf (Proxy @Int64)
withOptionalValue :: IO DuckDBValue -> (DuckDBValue -> IO a) -> IO (Maybe a)
withOptionalValue acquire action = do
value <- acquire
if value == coerce nullPtr
then pure Nothing
else do
result <- action value
destroy value
pure (Just result)
where
destroy val = alloca \ptr -> poke ptr val >> c_duckdb_destroy_value ptr
vectorDataPtr :: DuckDBVector -> IO (Ptr Int64)
vectorDataPtr vec = coerce <$> c_duckdb_vector_get_data vec
contains :: String -> String -> Bool
contains needle haystack = mapLower needle `isInfixOf` mapLower haystack
where
mapLower = map toLowerAscii
toLowerAscii c
| isAsciiUpper c = toEnum (fromEnum c + 32)
| otherwise = c
toCBool :: Bool -> CBool
toCBool False = CBool 0
toCBool True = CBool 1
peekCStringMaybe :: ConstPtr CChar -> IO String
peekCStringMaybe ptr
| ptr == coerce nullPtr = pure ""
| otherwise = peekCString (coerce ptr)
-- Wrappers -------------------------------------------------------------------
mkBindFun :: (DuckDBBindInfo -> IO ()) -> IO DuckDBTableFunctionBindFun
mkBindFun = fmap DuckDBTableFunctionBindFun . toFunPtr . DuckDBTableFunctionBindFun_Aux
mkInitFun :: (DuckDBInitInfo -> IO ()) -> IO DuckDBTableFunctionInitFun
mkInitFun = fmap DuckDBTableFunctionInitFun . toFunPtr . DuckDBTableFunctionInitFun_Aux
mkFunctionFun :: (DuckDBFunctionInfo -> DuckDBDataChunk -> IO ()) -> IO DuckDBTableFunctionFun
mkFunctionFun = fmap DuckDBTableFunctionFun . toFunPtr . DuckDBTableFunctionFun_Aux
mkDeleteCallback :: (Ptr Void -> IO ()) -> IO DuckDBDeleteCallback
mkDeleteCallback = fmap DuckDBDeleteCallback . toFunPtr . DuckDBDeleteCallback_Aux