duckdb-simple-1.0.0.0: src/Database/DuckDB/Simple/V2/Copy.hs
{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes #-}
-- |
-- Module : Database.DuckDB.Simple.V2.Copy
-- Description : Managed V2 COPY callbacks and multi-file table functions.
--
-- Callback info and context handles are borrowed and must not escape a callback.
-- The registration user-data slot is reserved for callback cleanup. Use closures
-- for registration data and 'setCallbackState' for bind, init, and batch state.
-- Transfer state to each slot once. Use an @IORef@ to change existing state.
-- COPY format callbacks require the threaded GHC runtime (@-threaded@).
-- Batch preparation may run concurrently. Flushes for one output file are serial.
-- Destroy a collection acquired with @copy_to_batch_take_input@ after use.
-- Multi-file functions use the callbacks of a registered single-file table
-- function.
--
-- Use this module when the row-oriented functions of "Database.DuckDB.Simple.Copy" are not sufficient, for
-- example to get a batch as a native column data collection, to set the batch size, or to give the
-- statistics for @RETURN_STATS@. Make a record of callbacks with 'copyToCallbacks' or 'copyFromCallbacks'.
-- Then register it with 'registerCopyToCallbacks' or 'registerCopyFromCallbacks', or register both sides
-- with 'registerCopyFunction'. 'registerMultiFileFunctionWith' adds file lists, globs, and hive partitions to
-- a registered single-file table function.
--
-- The examples use the imports and the definitions of the module header of
-- "Database.DuckDB.Simple.V2.Registration".
module Database.DuckDB.Simple.V2.Copy (
CopyToCallbacks (..),
copyToCallbacks,
CopyFromCallbacks (..),
copyFromCallbacks,
registerCopyFunction,
registerCopyFunctionWith,
registerCopyToCallbacks,
registerCopyFromCallbacks,
registerMultiFileFunctionWith,
RegistrationTarget (..),
ContextCallback,
setCallbackState,
getCallbackState,
) where
import Control.Monad (when)
import Data.Text (Text)
import qualified Data.Text as Text
import Database.DuckDB.Simple.Callback (
Allocator,
attachOpaque,
guardContext,
install,
installAll,
installOptional,
requireWorkerCallbackRuntime,
withCallbackResources,
withTarget,
)
import Database.DuckDB.Simple.Internal (Connection)
import Database.DuckDB.Simple.Native
import Database.DuckDB.Simple.V2.Registration (ContextCallback, RegistrationTarget (..), getCallbackState, setCallbackState)
-- | Every COPY TO callback in the pinned V2 API. Batch and flush are required.
data CopyToCallbacks = CopyToCallbacks
{ copyToBindCallback :: Maybe (ContextCallback DuckDBV2CopyToBindInfoHandle)
, copyToBatchSizeCallback :: Maybe (ContextCallback DuckDBV2CopyToBatchSizeInfoHandle)
, copyToInitCallback :: Maybe (ContextCallback DuckDBV2CopyToInitInfoHandle)
, copyToBatchCallback :: ContextCallback DuckDBV2CopyToBatchInfoHandle
, copyToFlushCallback :: ContextCallback DuckDBV2CopyToFlushInfoHandle
, copyToFinalizeCallback :: Maybe (ContextCallback DuckDBV2CopyToFinalizeInfoHandle)
, copyToStatisticsCallback :: Maybe (ContextCallback DuckDBV2CopyToStatisticsInfoHandle)
}
-- | Use batch and flush callbacks with no optional COPY TO callbacks.
copyToCallbacks :: ContextCallback DuckDBV2CopyToBatchInfoHandle -> ContextCallback DuckDBV2CopyToFlushInfoHandle -> CopyToCallbacks
copyToCallbacks batch flush = CopyToCallbacks Nothing Nothing Nothing batch flush Nothing Nothing
-- | Every COPY FROM callback in the pinned V2 API. Bind and exec are required.
data CopyFromCallbacks = CopyFromCallbacks
{ copyFromBindCallback :: ContextCallback DuckDBV2CopyFromBindInfoHandle
, copyFromInitGlobalCallback :: Maybe (ContextCallback DuckDBV2CopyFromInitGlobalInfoHandle)
, copyFromInitLocalCallback :: Maybe (ContextCallback DuckDBV2CopyFromInitLocalInfoHandle)
, copyFromExecCallback :: ContextCallback DuckDBV2CopyFromExecInfoHandle
, copyFromProgressCallback :: Maybe (ContextCallback DuckDBV2CopyFromProgressInfoHandle)
}
-- | Use bind and exec callbacks with no optional COPY FROM callbacks.
copyFromCallbacks :: ContextCallback DuckDBV2CopyFromBindInfoHandle -> ContextCallback DuckDBV2CopyFromExecInfoHandle -> CopyFromCallbacks
copyFromCallbacks bind execute = CopyFromCallbacks bind Nothing Nothing execute Nothing
-- | Register either direction or both. The library allocates the callback pointers. It gives their cleanup
-- to DuckDB in the user data of the native object. DuckDB runs the cleanup when it releases the object. See
-- Note [Callback ownership] in
-- <src/Database.DuckDB.Simple.Callback.html Database.DuckDB.Simple.Callback>.
registerCopyFunction :: Connection -> Text -> Maybe CopyToCallbacks -> Maybe CopyFromCallbacks -> IO ()
registerCopyFunction connection name to from = do
when (Text.null name || Text.any (== '\0') name) $ throwInvalidArgument "invalid copy function name"
registerCopyFunctionWith
(ConnectionTarget connection)
(\function -> withIdentifier name (checked . c_duckdb_v2_copy_function_set_name function))
to
from
-- | Register COPY callbacks on a connection or a borrowed extension target.
-- The configuration action sets the name of the format:
--
-- > V2Copy.registerCopyFunctionWith
-- > target
-- > (\function -> R.withIdentifier "managed_output" (R.checked . c_duckdb_v2_copy_function_set_name function))
-- > (Just callbacks)
-- > Nothing
registerCopyFunctionWith
:: RegistrationTarget -> (DuckDBV2CopyFunctionHandle -> IO ()) -> Maybe CopyToCallbacks -> Maybe CopyFromCallbacks -> IO ()
registerCopyFunctionWith target configure to from =
withTarget
target
c_duckdb_v2_copy_function_create
c_duckdb_v2_copy_function_destroy
c_duckdb_v2_connection_register_copy_function
c_duckdb_v2_extension_register_copy_function
\function register ->
withCallbackResources
(requireWorkerCallbackRuntime "COPY")
(\allocate -> installAll (toInstallers allocate function to <> fromInstallers allocate function from))
(attachOpaque (c_duckdb_v2_copy_function_set_user_data function))
\installCallbacks -> do
configure function
installCallbacks
register
-- | Register the advanced COPY TO callback interface.
--
-- The example registers the format @managed_output@, which counts the rows. The init callback makes the
-- output file and stores a counter as init data. The batch callback takes the rows of a batch and stores
-- their count as batch data. The flush callback adds that count to the counter. The example does not write
-- the rows. A format that writes rows converts them in the batch callback and writes them in the flush
-- callback.
--
-- > withConnection ":memory:" \connection -> do
-- > total <- newIORef (0 :: DuckDBIdx)
-- > let initialize info _ = do
-- > path <- R.output (c_duckdb_v2_copy_to_init_get_file_path info) >>= R.readText
-- > writeFile (Text.unpack path) ""
-- > R.setCallbackState (c_duckdb_v2_copy_to_init_set_init_data info) total
-- > batch info _ = do
-- > count <-
-- > bracket
-- > (R.output (c_duckdb_v2_copy_to_batch_take_input info))
-- > (R.destroy c_duckdb_v2_column_data_collection_destroy)
-- > (R.output . c_duckdb_v2_column_data_collection_row_count)
-- > R.setCallbackState (c_duckdb_v2_copy_to_batch_set_batch_data info) count
-- > flush info _ = do
-- > count <- R.getCallbackState (c_duckdb_v2_copy_to_flush_get_batch_data info) :: IO DuckDBIdx
-- > state <- R.getCallbackState (c_duckdb_v2_copy_to_flush_get_init_data info) :: IO (IORef DuckDBIdx)
-- > atomicModifyIORef' state (\n -> (n + count, ()))
-- > callbacks = (V2Copy.copyToCallbacks batch flush){V2Copy.copyToInitCallback = Just initialize}
-- > V2Copy.registerCopyToCallbacks connection "managed_output" callbacks
-- > _ <- execute_ connection "COPY (SELECT i FROM range(5000) t(i)) TO 'numbers.out' (FORMAT managed_output)"
-- > readIORef total >>= print -- DuckDBIdx {unwrap = 5000}
registerCopyToCallbacks :: Connection -> Text -> CopyToCallbacks -> IO ()
registerCopyToCallbacks connection name callbacks = registerCopyFunction connection name (Just callbacks) Nothing
-- | Register the advanced COPY FROM callback interface.
registerCopyFromCallbacks :: Connection -> Text -> CopyFromCallbacks -> IO ()
registerCopyFromCallbacks connection name callbacks = registerCopyFunction connection name Nothing (Just callbacks)
-- | Allocate the configured COPY TO callbacks before installing them.
toInstallers :: Allocator -> DuckDBV2CopyFunctionHandle -> Maybe CopyToCallbacks -> [IO (IO ())]
toInstallers _ _ Nothing = []
toInstallers allocate function (Just callbacks) =
[ installOptional (copyToBindCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyToBindCallbackFn
(DuckDBV2CopyToBindCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_to_set_bind_callback function)
, installOptional (copyToBatchSizeCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyToBatchSizeCallbackFn
(DuckDBV2CopyToBatchSizeCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_to_set_batch_size_callback function)
, installOptional (copyToInitCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyToInitCallbackFn
(DuckDBV2CopyToInitCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_to_set_init_callback function)
, install
allocate
DuckDBV2CopyToBatchCallbackFn
(DuckDBV2CopyToBatchCallbackFn_Aux (guardContext (copyToBatchCallback callbacks)))
(c_duckdb_v2_copy_to_set_batch_callback function)
, install
allocate
DuckDBV2CopyToFlushCallbackFn
(DuckDBV2CopyToFlushCallbackFn_Aux (guardContext (copyToFlushCallback callbacks)))
(c_duckdb_v2_copy_to_set_flush_callback function)
, installOptional (copyToFinalizeCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyToFinalizeCallbackFn
(DuckDBV2CopyToFinalizeCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_to_set_finalize_callback function)
, installOptional (copyToStatisticsCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyToStatisticsCallbackFn
(DuckDBV2CopyToStatisticsCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_to_set_statistics_callback function)
]
-- | Allocate the configured COPY FROM callbacks before installing them.
fromInstallers :: Allocator -> DuckDBV2CopyFunctionHandle -> Maybe CopyFromCallbacks -> [IO (IO ())]
fromInstallers _ _ Nothing = []
fromInstallers allocate function (Just callbacks) =
[ install
allocate
DuckDBV2CopyFromBindCallbackFn
(DuckDBV2CopyFromBindCallbackFn_Aux (guardContext (copyFromBindCallback callbacks)))
(c_duckdb_v2_copy_from_set_bind_callback function)
, installOptional (copyFromInitGlobalCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyFromInitGlobalCallbackFn
(DuckDBV2CopyFromInitGlobalCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_from_set_init_global_callback function)
, installOptional (copyFromInitLocalCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyFromInitLocalCallbackFn
(DuckDBV2CopyFromInitLocalCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_from_set_init_local_callback function)
, install
allocate
DuckDBV2CopyFromExecCallbackFn
(DuckDBV2CopyFromExecCallbackFn_Aux (guardContext (copyFromExecCallback callbacks)))
(c_duckdb_v2_copy_from_set_exec_callback function)
, installOptional (copyFromProgressCallback callbacks) $ \callback ->
install
allocate
DuckDBV2CopyFromProgressCallbackFn
(DuckDBV2CopyFromProgressCallbackFn_Aux (guardContext callback))
(c_duckdb_v2_copy_from_set_progress_callback function)
]
-- | Configure a multi-file wrapper on a connection or borrowed extension target.
-- 'Database.DuckDB.Simple.Copy.registerMultiFileFunction' sets the same fields without a native handle.
--
-- The example registers the table function @managed_single@, which reads the integer in one file. Then it
-- registers the multi-file function @managed_files@, which calls @managed_single@ for each file. The
-- configuration action sets the two names. DuckDB can run the exec callback of the single-file function on
-- more than one thread for one scan. Thus the exec callback takes its state with @atomicModifyIORef\'@.
--
-- > withConnection ":memory:" \connection -> do
-- > let bind input result context = do
-- > path <- bracket (R.output (c_duckdb_v2_function_bind_get_arg_value input 0)) (R.destroy c_duckdb_v2_value_destroy) \value ->
-- > R.output (c_duckdb_v2_value_get_varchar value) >>= R.readText
-- > value <- read <$> readFile (Text.unpack path) :: IO Int64
-- > R.setCallbackState (c_duckdb_v2_function_bind_set_bind_data input) value
-- > factory <- R.output (c_duckdb_v2_context_get_factory context)
-- > R.withTypeDeclaration factory bigint \dtype ->
-- > R.withIdentifier "value" \name -> R.checked (c_duckdb_v2_table_function_bind_add_result_column result name dtype)
-- > initialize info _ = do
-- > value <- R.getCallbackState (c_duckdb_v2_table_function_init_global_get_bind_data info)
-- > state <- newIORef (Just (value :: Int64))
-- > R.setCallbackState (c_duckdb_v2_table_function_init_global_set_global_state info) state
-- > executeRows info _ = do
-- > state <- R.getCallbackState (c_duckdb_v2_table_function_exec_get_global_state info) :: IO (IORef (Maybe Int64))
-- > value <- atomicModifyIORef' state (\previous -> (Nothing, previous))
-- > chunk <- R.output (c_duckdb_v2_table_function_exec_get_output_chunk info)
-- > vector <- R.output (c_duckdb_v2_data_chunk_get_vector chunk 0)
-- > writeIntegers vector (maybe [] pure value)
-- > R.registerTableFunction
-- > connection
-- > "managed_single"
-- > [R.parameter "path" (R.TypeExpression "VARCHAR")]
-- > (R.tableCallbacks bind executeRows){R.tableInitGlobalCallback = Just initialize}
-- > V2Copy.registerMultiFileFunctionWith (R.ConnectionTarget connection) \function -> do
-- > R.withText "managed_files" \(ConstPtr view) -> R.checked (c_duckdb_v2_multi_file_function_set_name function view)
-- > R.withText "managed_single" \(ConstPtr view) -> R.checked (c_duckdb_v2_multi_file_function_set_single_file_function function view)
-- > writeFile "first.mock" "7"
-- > writeFile "second.mock" "11"
-- > rows <- query_ connection "SELECT value FROM managed_files('*.mock') ORDER BY value"
-- > print (map fromOnly rows :: [Int64]) -- [7,11]
registerMultiFileFunctionWith :: RegistrationTarget -> (DuckDBV2MultiFileFunctionHandle -> IO ()) -> IO ()
registerMultiFileFunctionWith target configure =
withTarget
target
c_duckdb_v2_multi_file_function_create
c_duckdb_v2_multi_file_function_destroy
c_duckdb_v2_connection_register_multi_file_function
c_duckdb_v2_extension_register_multi_file_function
\function register -> do
configure function
register