packages feed

duckdb-simple-1.0.0.0: src/Database/DuckDB/Simple/Copy/Internal.hs

{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE ForeignFunctionInterface #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}

-- |
-- Module      : Database.DuckDB.Simple.Copy.Internal
-- Description : Share the typed COPY registrations for connections and extensions.
--
-- The cabal file lists this hidden module in @other-modules@.
-- The functions with the suffix @On@ take a registration target.
-- "Database.DuckDB.Simple.Copy" exports the connection forms.
-- "Database.DuckDB.Simple.Extension" exports the extension forms.
module Database.DuckDB.Simple.Copy.Internal (
  CopyBindInfo (..),
  LogicalTypeId (..),
  CopyInitInfo (..),
  CopySinkInfo (..),
  CopyFinalizeInfo (..),
  registerCopyToFunction,
  registerCopyToFunctionOn,
  CopyFromInfo (..),
  registerCopyFromFunction,
  registerCopyFromFunctionOn,
  CopyTo (..),
  registerCopyFunction,
  registerCopyFunctionOn,
  TypedRow,
  MultiFileOptions (..),
  defaultMultiFileOptions,
  registerMultiFileFunction,
  registerMultiFileFunctionOn,
) where

import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar)
import Control.Exception (bracket, finally, mask, mask_, onException)
import Control.Monad (forM, forM_, when)
import Data.Proxy (Proxy (..))
import Data.Text (Text)
import qualified Data.Text as Text
import Data.Void (Void)
import Database.DuckDB.Simple.Callback (attachOpaque, ignoreCallbackExceptions, readCallbackState, targetCastFactory, transferCallbackState)
import Database.DuckDB.Simple.FromField (Field (..), FieldValue)
import Database.DuckDB.Simple.Internal (Connection, withDatabaseHandle)
import Database.DuckDB.Simple.Materialize (
  FallbackReaders (..),
  ReadFactory,
  VectorReader,
  callbackReadFactory,
  prepareGeometryReader,
  prepareVectorReader,
  readCallbackVariant,
  readFactoryHandle,
  readInstanceVariant,
 )
import Database.DuckDB.Simple.Native
import Database.DuckDB.Simple.TypeScope (scopedTypeRep, withTypeScope)
import Database.DuckDB.Simple.Typed (TypedRow (..))
import Database.DuckDB.Simple.Typed.Table (writeRows)
import Database.DuckDB.Simple.V2.Copy (
  CopyFromCallbacks (..),
  CopyToCallbacks (..),
  RegistrationTarget (..),
  copyFromCallbacks,
  copyToCallbacks,
  registerCopyFunctionWith,
  registerMultiFileFunctionWith,
 )
import Database.DuckDB.Simple.V2.LogicalRep (logicalTypeToRep)
import Foreign.C.ConstPtr (ConstPtr (..))
import Foreign.Marshal.Array (withArray)
import Foreign.Ptr (FunPtr, Ptr, castFunPtr, castPtr, nullPtr)
import Foreign.StablePtr (castPtrToStablePtr, castStablePtrToPtr, deRefStablePtr, freeStablePtr, newStablePtr)

-- | Bind-phase metadata for a custom COPY TO function.
data CopyBindInfo = CopyBindInfo
  { copyBindColumnTypes :: ![LogicalTypeId]
  }
  deriving (Eq, Show)

-- | Init-phase inputs for one output file or partition.
data CopyInitInfo bindState = CopyInitInfo
  { copyInitBindState :: !bindState
  , copyInitFilePath :: !FilePath
  }

-- | Flush-phase state passed to the row-oriented sink callback.
data CopySinkInfo bindState globalState = CopySinkInfo
  { copySinkBindState :: !bindState
  , copySinkGlobalState :: !globalState
  }

-- | Finalize-phase inputs for one output file or partition.
data CopyFinalizeInfo bindState globalState = CopyFinalizeInfo
  { copyFinalizeBindState :: !bindState
  , copyFinalizeGlobalState :: !globalState
  }

-- | Register row-oriented COPY TO callbacks. Native batch buffers are copied.
-- Use 'registerCopyFunction' for a format with a COPY FROM side too. DuckDB keeps
-- the first registration of a format name. A later registration of the same name
-- has no effect and gives no error.
registerCopyToFunction
  :: Connection
  -> Text
  -> (CopyBindInfo -> IO bindState)
  -> (CopyInitInfo bindState -> IO globalState)
  -> (CopySinkInfo bindState globalState -> [[Field]] -> IO ())
  -> (CopyFinalizeInfo bindState globalState -> IO ())
  -> IO ()
registerCopyToFunction = registerCopyToFunctionOn . ConnectionTarget

-- | Register row-oriented COPY TO callbacks on a connection or a loading
-- extension. The rules of 'registerCopyToFunction' apply.
registerCopyToFunctionOn
  :: RegistrationTarget
  -> Text
  -> (CopyBindInfo -> IO bindState)
  -> (CopyInitInfo bindState -> IO globalState)
  -> (CopySinkInfo bindState globalState -> [[Field]] -> IO ())
  -> (CopyFinalizeInfo bindState globalState -> IO ())
  -> IO ()
registerCopyToFunctionOn target name bindFn initFn sinkFn finalizeFn = do
  checkCopyName name
  casts <- targetCastFactory target
  withVariantReader target \readVariant ->
    registerCopyOn target name (Just (copyToSide casts readVariant bindFn initFn sinkFn finalizeFn)) Nothing

-- | Check the format name and register the native callbacks of each side.
registerCopyOn :: RegistrationTarget -> Text -> Maybe CopyToCallbacks -> Maybe CopyFromCallbacks -> IO ()
registerCopyOn target name to from = do
  checkCopyName name
  registerCopyFunctionWith target (\function -> withIdentifier name (checked . c_duckdb_v2_copy_function_set_name function)) to from

-- | Raise the input error for a format name that is empty or contains NUL. A
-- COPY TO side checks the name before the read of the state of the target.
checkCopyName :: Text -> IO ()
checkCopyName name = when (Text.null name || Text.any (== '\0') name) $ throwInvalidArgument "invalid copy function name"

-- | Use the host instance for a connection's COPY TO and a private one for an extension.
withVariantReader :: RegistrationTarget -> (VectorReader -> IO a) -> IO a
withVariantReader target action = case target of
  -- A catalog-owned callback must not keep the managed instance alive.
  ConnectionTarget connection -> withDatabaseHandle connection (action . readInstanceVariant)
  ExtensionTarget _ _ -> action readCallbackVariant

-- | The native COPY TO callbacks of the row-oriented functions. The first
-- argument is the factory of 'targetCastFactory'.
copyToSide
  :: forall bindState globalState
   . Maybe DuckDBV2FactoryHandle
  -> VectorReader
  -> (CopyBindInfo -> IO bindState)
  -> (CopyInitInfo bindState -> IO globalState)
  -> (CopySinkInfo bindState globalState -> [[Field]] -> IO ())
  -> (CopyFinalizeInfo bindState globalState -> IO ())
  -> CopyToCallbacks
copyToSide casts readVariant bindFn initFn sinkFn finalizeFn =
  (copyToCallbacks batch flush)
    { copyToBindCallback = Just bind
    , copyToInitCallback = Just initialize
    , copyToFinalizeCallback = Just finalize
    }
  where
    bind info _ = mask \restore -> do
      count <- output (c_duckdb_v2_copy_to_bind_get_column_count info)
      types <- acquireItems destroyLogicalType [output (c_duckdb_v2_copy_to_bind_get_column_type info index) | index <- indices count]
      state <-
        restore (mapM (\logical -> output (c_duckdb_v2_logical_type_get_id logical) >>= logicalTypeIdFromNative) types >>= bindFn . CopyBindInfo)
          `onException` releaseItems destroyLogicalType types
      stable <- newStablePtr (state, types) `onException` releaseItems destroyLogicalType types
      attachOpaque (c_duckdb_v2_copy_to_bind_set_bind_data info) (castPtr (castStablePtrToPtr stable)) copyBindingDestructor
        `onException` (freeStablePtr stable >> releaseItems destroyLogicalType types)
    initialize info _ = do
      (state, _) <- readCallbackState (c_duckdb_v2_copy_to_init_get_bind_data info)
      path <- output (c_duckdb_v2_copy_to_init_get_file_path info) >>= readText
      global <- initFn (CopyInitInfo state (Text.unpack path))
      transferCallbackState (attachOpaque (c_duckdb_v2_copy_to_init_set_init_data info)) global
    batch info context = withContextFactory context $ \factory -> do
      (_, types) <- readCallbackState (c_duckdb_v2_copy_to_batch_get_bind_data info) :: IO (bindState, [DuckDBV2LogicalTypeHandle])
      rows <-
        bracket
          (output (c_duckdb_v2_copy_to_batch_take_input info))
          (destroy c_duckdb_v2_column_data_collection_destroy)
          (materializeCollection (callbackReadFactory casts factory) readVariant types)
      transferCallbackState (attachOpaque (c_duckdb_v2_copy_to_batch_set_batch_data info)) rows
    flush info context = withTypeScope \scope -> withContextFactory context \factory -> do
      (state, types) <- readCallbackState (c_duckdb_v2_copy_to_flush_get_bind_data info)
      global <- readCallbackState (c_duckdb_v2_copy_to_flush_get_init_data info)
      rows <- readCallbackState (c_duckdb_v2_copy_to_flush_get_batch_data info)
      -- Bind data owns the types through this callback. Copy them into the
      -- flush scope. See Note [Scoped field types] in
      -- "Database.DuckDB.Simple.TypeScope".
      columnTypes <- mapM (scopedTypeRep scope (pure factory) . output . c_duckdb_v2_logical_type_copy) types
      let fields = zipWith3 (\index logical value -> Field (Text.pack ("column" <> show index)) index value logical) [0 ..] columnTypes
      sinkFn (CopySinkInfo state global) (map fields rows)
    finalize info _ = do
      (state, _) <- readCallbackState (c_duckdb_v2_copy_to_finalize_get_bind_data info)
      global <- readCallbackState (c_duckdb_v2_copy_to_finalize_get_init_data info)
      finalizeFn (CopyFinalizeInfo state global)

-- | Release native bind descriptors before their database closes. See
-- Note [Callback ownership] in "Database.DuckDB.Simple.Callback" for the rules
-- of a destructor.
releaseCopyBinding :: Ptr Void -> IO ()
releaseCopyBinding raw =
  mask_
    $ ignoreCallbackExceptions
    $ if raw == nullPtr
      then pure ()
      else do
        let stable = castPtrToStablePtr (castPtr raw)
        (deRefStablePtr stable >>= \(_, types) -> releaseItems destroyLogicalType types)
          `finally` freeStablePtr stable

foreign export ccall "duckdb_simple_release_copy_binding" releaseCopyBinding :: Ptr Void -> IO ()

foreign import ccall "&duckdb_simple_release_copy_binding" copyBindingDestructorAddress :: FunPtr (Ptr Void -> IO ())

-- | Use a static destructor address for native bind-state ownership.
copyBindingDestructor :: DuckDBV2OpaqueDestroyFn
copyBindingDestructor = DuckDBV2OpaqueDestroyFn (castFunPtr copyBindingDestructorAddress)

-- | Copy each scanned chunk before the scan invalidates its borrowed buffers.
-- The factory makes the chunk and reads GEOMETRY.
materializeCollection
  :: ReadFactory -> VectorReader -> [DuckDBV2LogicalTypeHandle] -> DuckDBV2ColumnDataCollectionHandle -> IO [[FieldValue]]
materializeCollection readFactory readVariant types collection =
  withArray types \typePtr ->
    bracket
      (output (c_duckdb_v2_data_chunk_create (readFactoryHandle readFactory) (ConstPtr typePtr) (fromIntegral (length types))))
      (destroy c_duckdb_v2_data_chunk_destroy)
      \chunk ->
        bracket
          (output (c_duckdb_v2_column_data_collection_shared_scan_state_create collection))
          (destroy c_duckdb_v2_column_data_collection_shared_scan_state_destroy)
          \shared ->
            bracket
              (output (c_duckdb_v2_column_data_collection_worker_scan_state_create collection))
              (destroy c_duckdb_v2_column_data_collection_worker_scan_state_destroy)
              \worker ->
                let loop groups = do
                      produced <- output (c_duckdb_v2_column_data_collection_scan collection shared worker chunk)
                      if produced == 0
                        then pure (concat (reverse groups))
                        else do
                          rows <-
                            materializeChunkRows
                              (FallbackReaders (logicalTypeToRep (readFactoryHandle readFactory)) readVariant (prepareGeometryReader readFactory))
                              chunk
                          loop (rows : groups)
                in  loop []

-- | Copy the values of a complete chunk before its collection scan advances.
-- The flush callback adds the fields and their scoped type actions.
materializeChunkRows :: FallbackReaders -> DuckDBV2DataChunkHandle -> IO [[FieldValue]]
materializeChunkRows readers chunk = do
  columns <- output (c_duckdb_v2_data_chunk_get_vector_count chunk)
  columnReaders <- forM [0 .. fromIntegral columns - 1] \index -> do
    vector <- output (c_duckdb_v2_data_chunk_get_vector chunk (fromIntegral (index :: Int)))
    prepareVectorReader readers vector
  rows <- output (c_duckdb_v2_data_chunk_get_size chunk)
  forM [0 .. fromIntegral rows - 1] \row -> mapM ($ row) columnReaders

-- | The inputs of the COPY FROM side for one statement. The path is the text
-- of the statement. DuckDB does not expand a glob pattern in it. The names and
-- the types are the columns of the target table, in order.
data CopyFromInfo = CopyFromInfo
  { copyFromPath :: !FilePath
  , copyFromColumnNames :: ![Text]
  , copyFromColumnTypes :: ![LogicalTypeId]
  }
  deriving (Eq, Show)

-- | Register the COPY FROM side of a format. Use 'registerCopyFunction' for a
-- format with the two sides. DuckDB keeps the first registration of a format
-- name. A later registration of the same name has no effect and gives no error.
--
-- The function gets the path and the columns of the target table. It gives the
-- rows to insert. It runs one time for each run of a COPY FROM statement. DuckDB
-- takes the rows from the list one chunk at a time, so the list can be lazy. A
-- row must have one column for each column of the table, in the same order. An
-- exception from the function or from a row fails the statement.
--
-- > registerCopyFromFunction conn "lines" \info -> map Only . Text.lines <$> TextIO.readFile (copyFromPath info)
registerCopyFromFunction :: (TypedRow row) => Connection -> Text -> (CopyFromInfo -> IO [row]) -> IO ()
registerCopyFromFunction = registerCopyFromFunctionOn . ConnectionTarget

-- | Register the COPY FROM side of a format on a connection or a loading
-- extension. The rules of 'registerCopyFromFunction' apply.
registerCopyFromFunctionOn :: (TypedRow row) => RegistrationTarget -> Text -> (CopyFromInfo -> IO [row]) -> IO ()
registerCopyFromFunctionOn target name readRows = registerCopyOn target name Nothing (Just (copyFromSide name readRows))

-- | The native COPY FROM callbacks of a typed reader. The bind callback reads
-- the path and the columns of the target table. See
-- Note [Row lists of typed scans] in "Database.DuckDB.Simple.Typed.Table".
copyFromSide :: forall row. (TypedRow row) => Text -> (CopyFromInfo -> IO [row]) -> CopyFromCallbacks
copyFromSide name readRows = (copyFromCallbacks bind execute){copyFromInitGlobalCallback = Just initialize}
  where
    width = length (rowColumnDeclarations (Proxy @row))
    bind info _ = do
      path <- output (c_duckdb_v2_copy_from_bind_get_file_path info) >>= readText
      count <- output (c_duckdb_v2_copy_from_bind_get_column_count info)
      when (fromIntegral count /= width) $
        throwInvalidArgument
          ( "the table of COPY FROM has "
              <> Text.pack (show (fromIntegral count :: Int))
              <> " columns, but a row of "
              <> name
              <> " has "
              <> Text.pack (show width)
          )
      let columns = takeWhile (< count) [0 ..]
      names <- forM columns \index -> output (c_duckdb_v2_copy_from_bind_get_column_name info index) >>= readIdentifier
      types <- forM columns \index ->
        bracket
          (output (c_duckdb_v2_copy_from_bind_get_column_type info index))
          destroyLogicalType
          (\logical -> output (c_duckdb_v2_logical_type_get_id logical) >>= logicalTypeIdFromNative)
      transferCallbackState
        (attachOpaque (c_duckdb_v2_copy_from_bind_set_bind_data info))
        (readRows (CopyFromInfo (Text.unpack path) names types))
    initialize info _ = do
      scan <- readCallbackState (c_duckdb_v2_copy_from_init_global_get_bind_data info)
      remaining <- scan >>= newMVar :: IO (MVar [row])
      transferCallbackState (attachOpaque (c_duckdb_v2_copy_from_init_global_set_global_state info)) remaining
    execute info context = withContextFactory context \factory -> do
      remaining <- readCallbackState (c_duckdb_v2_copy_from_exec_get_global_state info) :: IO (MVar [row])
      chunk <- output (c_duckdb_v2_copy_from_exec_get_output_chunk info)
      -- The lock gives each row to one call only. See
      -- Note [Row lists of typed scans] in "Database.DuckDB.Simple.Typed.Table".
      modifyMVar_ remaining (writeRows factory chunk (take width [0 ..]))

-- | The COPY TO side of a format. The four functions have the roles of the
-- four functions of 'registerCopyToFunction'.
data CopyTo bindState globalState = CopyTo
  { copyToBind :: CopyBindInfo -> IO bindState
  , copyToInit :: CopyInitInfo bindState -> IO globalState
  , copyToSink :: CopySinkInfo bindState globalState -> [[Field]] -> IO ()
  , copyToFinalize :: CopyFinalizeInfo bindState globalState -> IO ()
  }

-- | Register a format with a COPY TO side and a COPY FROM side under one
-- name. The rules of 'registerCopyToFunction' apply to the COPY TO side. The
-- rules of 'registerCopyFromFunction' apply to the COPY FROM side. Two
-- registrations of one name do not give a format with the two sides, because
-- DuckDB keeps the first registration.
registerCopyFunction :: (TypedRow row) => Connection -> Text -> CopyTo bindState globalState -> (CopyFromInfo -> IO [row]) -> IO ()
registerCopyFunction = registerCopyFunctionOn . ConnectionTarget

-- | Register a format with the two sides on a connection or a loading
-- extension. The rules of 'registerCopyFunction' apply.
registerCopyFunctionOn
  :: (TypedRow row) => RegistrationTarget -> Text -> CopyTo bindState globalState -> (CopyFromInfo -> IO [row]) -> IO ()
registerCopyFunctionOn target name (CopyTo bindFn initFn sinkFn finalizeFn) readRows = do
  checkCopyName name
  casts <- targetCastFactory target
  withVariantReader target \readVariant ->
    registerCopyOn target name (Just (copyToSide casts readVariant bindFn initFn sinkFn finalizeFn)) (Just (copyFromSide name readRows))

-- | Options supplied to DuckDB's multi-file wrapper.
data MultiFileOptions = MultiFileOptions
  { multiFileReaderType :: Maybe Text
  , multiFileExtension :: Maybe Text
  }
  deriving (Eq, Show)

-- | Use native defaults for reader descriptions and directory expansion.
defaultMultiFileOptions :: MultiFileOptions
defaultMultiFileOptions = MultiFileOptions Nothing Nothing

-- | Register globbing, file lists, filename columns, and hive partition
-- support for a registered single-file table function. The second name is that
-- table function. DuckDB can run the exec callback of the single-file function on
-- more than one thread at the same time for one scan. A function from
-- 'Database.DuckDB.Simple.Function.createTableFunction' permits that. A table
-- function with native callbacks must synchronize its scan state.
registerMultiFileFunction :: Connection -> Text -> Text -> MultiFileOptions -> IO ()
registerMultiFileFunction connection = registerMultiFileFunctionOn (ConnectionTarget connection)

-- | Register a multi-file wrapper on a connection or a loading extension. The
-- second name is the registered single-file table function. Both names must not
-- be empty and must not contain NUL.
registerMultiFileFunctionOn :: RegistrationTarget -> Text -> Text -> MultiFileOptions -> IO ()
registerMultiFileFunctionOn target name singleFile MultiFileOptions{multiFileReaderType, multiFileExtension} = do
  when (any (\value -> Text.null value || Text.any (== '\0') value) [name, singleFile]) $
    throwInvalidArgument "invalid multi-file function name"
  registerMultiFileFunctionWith target \function -> do
    withText name (\(ConstPtr view) -> checked (c_duckdb_v2_multi_file_function_set_name function view))
    withText singleFile (\(ConstPtr view) -> checked (c_duckdb_v2_multi_file_function_set_single_file_function function view))
    forM_ multiFileReaderType \value -> withText value (\(ConstPtr view) -> checked (c_duckdb_v2_multi_file_function_set_reader_type function view))
    forM_ multiFileExtension \value -> withText value (\(ConstPtr view) -> checked (c_duckdb_v2_multi_file_function_set_file_extension function view))