duckdb-simple-1.0.0.0: test/CopyTests.hs
{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ScopedTypeVariables #-}
-- |
-- Module : CopyTests
-- Description : Native integration checks for managed COPY and multi-file callbacks.
module CopyTests (copyTests) where
import Control.Exception (bracket, try)
import Control.Monad (forM_, void)
import Data.IORef
import Data.Int (Int64)
import Data.List (sort)
import Data.Text (Text)
import qualified Data.Text as Text
import Database.DuckDB.FFI.V2
import Database.DuckDB.Simple hiding (bind)
import qualified Database.DuckDB.Simple.Catalog as Catalog
import qualified Database.DuckDB.Simple.Copy as Copy
import Database.DuckDB.Simple.FromField (Field (..), FieldValue (..), LogicalTypeRep (..))
import Database.DuckDB.Simple.LogicalRep (withoutAliases)
import Database.DuckDB.Simple.Ok (Ok (..))
import qualified Database.DuckDB.Simple.V2.Copy as V2Copy
import qualified Database.DuckDB.Simple.V2.Registration as R
import Database.DuckDB.Simple.Variant (variantObject)
import Foreign.C.ConstPtr (ConstPtr (..))
import Foreign.Ptr (nullPtr)
import System.Directory (createDirectory, getTemporaryDirectory, removeFile, removePathForcibly)
import System.FilePath ((</>))
import System.IO (hClose, openTempFile)
import System.Mem (performMajorGC)
import Test.Tasty (TestTree, testGroup)
import Test.Tasty.HUnit
import TestUtils (writeIntegers)
-- | Exercise the available COPY directions and multi-file wrapper.
copyTests :: TestTree
copyTests =
testGroup
"managed COPY"
[ testCase "COPY TO scans batches and reports statistics" testCopyTo
, testCase "row callbacks receive per-partition file paths" testPartitionedCopy
, testCase "row callbacks decode VARIANT after the registering connection closes" testVariantCopy
, testCase "COPY FROM runs bind global local and exec callbacks" testCopyFrom
, testCase "callback exceptions become SQL errors and permit reuse" testCopyFailure
, testCase "multi-file wrappers support lists globs filename and hive columns" testMultiFile
, testCase "COPY TO and table descriptions report the logical type id of each column type" testColumnTypeIds
, testCase "COPY TO snapshots survive but native metadata expires after the callback" testCopyFieldTypes
]
-- | Run every COPY TO callback and check the reported file statistics.
testCopyTo :: Assertion
testCopyTo = withDirectory \directory -> withConnection ":memory:" \connection -> do
seen <- newIORef ([] :: [Text])
total <- newIORef (0 :: DuckDBIdx)
let mark phase = atomicModifyIORef' seen (\phases -> (phase : phases, ()))
bind info _ = do
mark "bind"
R.output (c_duckdb_v2_copy_to_bind_get_column_count info) >>= (@?= 1)
R.setCallbackState (c_duckdb_v2_copy_to_bind_set_bind_data info) ()
batchSize info _ = mark "batch size" >> R.checked (c_duckdb_v2_copy_to_batch_size_set_target info 1024)
initialize info _ = do
mark "init"
path <- R.output (c_duckdb_v2_copy_to_init_get_file_path info) >>= R.readText
writeFile (Text.unpack path) ""
R.setCallbackState (c_duckdb_v2_copy_to_init_set_init_data info) total
batch info _ = do
mark "batch"
count <-
bracket
(R.output (c_duckdb_v2_copy_to_batch_take_input info))
(R.destroy c_duckdb_v2_column_data_collection_destroy)
(R.output . c_duckdb_v2_column_data_collection_row_count)
R.setCallbackState (c_duckdb_v2_copy_to_batch_set_batch_data info) count
flush info _ = do
mark "flush"
count <- R.getCallbackState (c_duckdb_v2_copy_to_flush_get_batch_data info) :: IO DuckDBIdx
state <- R.getCallbackState (c_duckdb_v2_copy_to_flush_get_init_data info) :: IO (IORef DuckDBIdx)
atomicModifyIORef' state (\n -> (n + count, ()))
finalize _ _ = mark "finalize"
statistics info _ = do
mark "statistics"
state <- R.getCallbackState (c_duckdb_v2_copy_to_statistics_get_init_data info) :: IO (IORef DuckDBIdx)
count <- readIORef state
R.checked (c_duckdb_v2_copy_to_statistics_set_row_count info count)
R.checked (c_duckdb_v2_copy_to_statistics_set_file_size info 17)
callbacks =
(V2Copy.copyToCallbacks batch flush)
{ V2Copy.copyToBindCallback = Just bind
, V2Copy.copyToBatchSizeCallback = Just batchSize
, V2Copy.copyToInitCallback = Just initialize
, V2Copy.copyToFinalizeCallback = Just finalize
, V2Copy.copyToStatisticsCallback = Just statistics
}
V2Copy.registerCopyToCallbacks connection "managed_output" callbacks
rows <-
query
connection
"COPY (SELECT i FROM range(5000) t(i)) TO ? (FORMAT managed_output, RETURN_STATS)"
(Only (Text.pack (directory </> "stats")))
:: IO [[FieldValue]]
readIORef total >>= (@?= 5000)
phases <- readIORef seen
forM_ ["bind", "batch size", "init", "batch", "flush", "finalize", "statistics"] \phase -> assertBool (Text.unpack phase) (phase `elem` phases)
case rows of
[FieldText _ : count : size : _] -> do
number count @?= 5000
number size @?= 17
_ -> assertFailure (show rows)
-- | Preserve the row-oriented API while native COPY creates partition files.
testPartitionedCopy :: Assertion
testPartitionedCopy = withDirectory \directory -> withConnection ":memory:" \connection -> do
files <- newIORef ([] :: [FilePath])
values <- newIORef ([] :: [Int64])
Copy.registerCopyToFunction
connection
"managed_partitions"
(\info -> Copy.copyBindColumnTypes info @?= [Copy.LogicalTypeIdBigInt])
( \info -> do
let path = Copy.copyInitFilePath info
writeFile path ""
atomicModifyIORef' files (\paths -> (path : paths, ()))
pure path
)
( \info rows -> do
let decoded = [value | [Field{fieldValue = FieldInt64 value}] <- rows]
atomicModifyIORef' values (\previous -> (decoded <> previous, ()))
appendFile (Copy.copySinkGlobalState info) (unlines (map show decoded))
)
(const (pure ()))
void
( execute
connection
"COPY (SELECT i, i%2 AS part FROM range(12) t(i)) TO ? (FORMAT managed_partitions, PARTITION_BY(part))"
(Only (Text.pack (directory </> "partitioned")))
)
readIORef values >>= (@?= [0 .. 11]) . sort
paths <- readIORef files
length paths @?= 2
assertBool "partition paths" (all (Text.isInfixOf "part=" . Text.pack) paths)
-- | COPY TO keeps pure type snapshots and expires escaped native actions.
testCopyFieldTypes :: Assertion
testCopyFieldTypes = withDirectory \directory -> withConnection ":memory:" \connection -> do
received <- newIORef ([] :: [Field])
snapshots <- newIORef ([] :: [LogicalTypeRep])
Copy.registerCopyToFunction
connection
"field_types"
(const (pure ()))
(\info -> writeFile (Copy.copyInitFilePath info) "")
( \_ rows -> do
forM_ rows \fields -> case fields of
[decimal, json, _] -> do
runConversion (traverse fieldType [decimal, json]) >>= \case
Ok types -> atomicModifyIORef' snapshots (\previous -> (previous <> types, ()))
Errors errors -> assertFailure (show errors)
_ -> assertFailure "expected three COPY TO columns"
atomicModifyIORef' received (\previous -> (previous <> concat rows, ()))
)
(const (pure ()))
void
( execute
connection
"COPY (SELECT 1.5::DECIMAL(4,1) AS d, '{}'::JSON AS j, 7::BIGINT AS unused) TO ? (FORMAT field_types)"
(Only (Text.pack (directory </> "types")))
)
performMajorGC
readIORef snapshots >>= (@?= [LogicalTypeDecimal 4 1, LogicalTypeAlias "JSON" (LogicalTypeScalar LogicalTypeIdVarchar)])
readIORef received >>= mapM_ \entry ->
try (runConversion (fieldType entry)) >>= \case
Left SQLError{sqlErrorMessage} -> sqlErrorMessage @?= "duckdb-simple: field type scope is closed"
Right _ -> assertFailure "COPY TO metadata survived its callback"
-- | COPY registrations share the instance and copy nested VARIANT payloads.
testVariantCopy :: Assertion
testVariantCopy = withDirectory \directory -> withEnvironment \environment -> withInstance environment [] \instance' -> do
received <- newIORef ([] :: [FieldValue])
withConnectionOn instance' \connection ->
Copy.registerCopyToFunction
connection
"managed_variant"
(\info -> Copy.copyBindColumnTypes info @?= [Copy.LogicalTypeIdVariant])
(\info -> writeFile (Copy.copyInitFilePath info) "")
(\_ rows -> atomicModifyIORef' received (\previous -> (previous <> [fieldValue entry | [entry] <- rows], ())))
(const (pure ()))
withConnectionOn instance' \connection -> do
void
( execute
connection
"COPY (SELECT {'n': i, 'values': [i, NULL]}::VARIANT FROM range(3) t(i)) TO ? (FORMAT managed_variant)"
(Only (Text.pack (directory </> "variant")))
)
readIORef received >>= (@?= [variantObject [("n", FieldInt64 i), ("values", FieldList [FieldInt64 i, FieldNull])] | i <- [0 .. 2]])
-- | Create a column of each type that DuckDB permits. Check the id that the
-- COPY TO bind info reports for each column. Check the id in the table
-- description of each column that has a scalar type. Check that no column type
-- in the table description has an alias, so the name of each id in
-- logicalTypeIdName is the name that DuckDB gives.
testColumnTypeIds :: Assertion
testColumnTypeIds = withDirectory \directory -> withConnection ":memory:" \connection -> do
reported <- newIORef []
Copy.registerCopyToFunction
connection
"column_types"
(writeIORef reported . Copy.copyBindColumnTypes)
(\info -> writeFile (Copy.copyInitFilePath info) "")
(\_ _ -> pure ())
(const (pure ()))
let named = zip [Text.pack ("c" <> show index) | index <- [0 :: Int ..]] tableColumns
void (execute_ connection (Query ("CREATE TABLE typed (" <> Text.intercalate ", " [name <> " " <> sql | (name, (sql, _)) <- named] <> ")")))
-- A table cannot have a TYPE column. A query can.
void (execute connection "COPY (SELECT NULL::TYPE, * FROM typed) TO ? (FORMAT column_types)" (Only (Text.pack (directory </> "types"))))
readIORef reported >>= (@?= LogicalTypeIdType : map snd tableColumns)
sort (LogicalTypeIdType : map snd tableColumns) @?= filter (`notElem` unreachable) [minBound .. maxBound]
description <- Catalog.describeTable connection "typed"
[ (name, ident)
| Catalog.ColumnDescription{Catalog.columnDescriptionName = name, Catalog.columnDescriptionType = LogicalTypeScalar ident} <-
Catalog.tableDescriptionColumns description
]
@?= [(name, ident) | (name, (_, ident)) <- named, ident `notElem` nested]
let types = map Catalog.columnDescriptionType (Catalog.tableDescriptionColumns description)
map withoutAliases types @?= types
where
-- DuckDB uses these ids for an unset type, an unresolved parameter, and a
-- function parameter of any type. SQL cannot declare a column of these types.
unreachable = [LogicalTypeIdInvalid, LogicalTypeIdUnknown, LogicalTypeIdAny]
-- LogicalTypeRep has a constructor with the type parameters for these ids.
nested =
[ LogicalTypeIdDecimal
, LogicalTypeIdGeometry
, LogicalTypeIdStruct
, LogicalTypeIdList
, LogicalTypeIdMap
, LogicalTypeIdEnum
, LogicalTypeIdUnion
, LogicalTypeIdArray
, LogicalTypeIdTuple
]
tableColumns =
[ ("\"NULL\"", LogicalTypeIdSQLNull)
, ("BOOLEAN", LogicalTypeIdBoolean)
, ("TINYINT", LogicalTypeIdTinyInt)
, ("SMALLINT", LogicalTypeIdSmallInt)
, ("INTEGER", LogicalTypeIdInteger)
, ("BIGINT", LogicalTypeIdBigInt)
, ("DATE", LogicalTypeIdDate)
, ("TIME", LogicalTypeIdTime)
, ("TIMESTAMP_S", LogicalTypeIdTimestampSec)
, ("TIMESTAMP_MS", LogicalTypeIdTimestampMs)
, ("TIMESTAMP", LogicalTypeIdTimestamp)
, ("TIMESTAMP_NS", LogicalTypeIdTimestampNs)
, ("DECIMAL(10, 2)", LogicalTypeIdDecimal)
, ("FLOAT", LogicalTypeIdFloat)
, ("DOUBLE", LogicalTypeIdDouble)
, ("VARCHAR", LogicalTypeIdVarchar)
, ("BLOB", LogicalTypeIdBlob)
, ("INTERVAL", LogicalTypeIdInterval)
, ("UTINYINT", LogicalTypeIdUTinyInt)
, ("USMALLINT", LogicalTypeIdUSmallInt)
, ("UINTEGER", LogicalTypeIdUInteger)
, ("UBIGINT", LogicalTypeIdUBigInt)
, ("TIMESTAMPTZ", LogicalTypeIdTimestampTz)
, ("TIMESTAMPTZ_NS", LogicalTypeIdTimestampTzNs)
, ("TIMETZ", LogicalTypeIdTimeTz)
, ("TIME_NS", LogicalTypeIdTimeNs)
, ("BIT", LogicalTypeIdBit)
, ("BIGNUM", LogicalTypeIdBigNum)
, ("UHUGEINT", LogicalTypeIdUHugeInt)
, ("HUGEINT", LogicalTypeIdHugeInt)
, ("UUID", LogicalTypeIdUUID)
, ("GEOMETRY", LogicalTypeIdGeometry)
, ("STRUCT(a INTEGER)", LogicalTypeIdStruct)
, ("INTEGER[]", LogicalTypeIdList)
, ("MAP(INTEGER, VARCHAR)", LogicalTypeIdMap)
, ("ENUM('a', 'b')", LogicalTypeIdEnum)
, ("UNION(a INTEGER, b VARCHAR)", LogicalTypeIdUnion)
, ("INTEGER[2]", LogicalTypeIdArray)
, ("VARIANT", LogicalTypeIdVariant)
, ("TUPLE(INTEGER, VARCHAR)", LogicalTypeIdTuple)
]
-- | Fill one output vector from state created in each native COPY FROM phase.
testCopyFrom :: Assertion
testCopyFrom = withConnection ":memory:" \connection -> do
phases <- newIORef ([] :: [Text])
let mark phase = atomicModifyIORef' phases (\previous -> (phase : previous, ()))
bind info _ = do
mark "bind"
R.output (c_duckdb_v2_copy_from_bind_get_column_count info) >>= (@?= 1)
R.setCallbackState (c_duckdb_v2_copy_from_bind_set_bind_data info) (7 :: Int64)
R.checked (c_duckdb_v2_copy_from_bind_set_cardinality info 3 1)
initializeGlobal info _ = do
mark "global"
start <- R.getCallbackState (c_duckdb_v2_copy_from_init_global_get_bind_data info)
state <- newIORef (Just (start :: Int64))
R.setCallbackState (c_duckdb_v2_copy_from_init_global_set_global_state info) state
R.checked (c_duckdb_v2_copy_from_init_global_set_max_threads info 1)
initializeLocal info _ = do
mark "local"
R.setCallbackState (c_duckdb_v2_copy_from_init_local_set_local_state info) ()
executeRows info _ = do
mark "exec"
() <- R.getCallbackState (c_duckdb_v2_copy_from_exec_get_local_state info)
state <- R.getCallbackState (c_duckdb_v2_copy_from_exec_get_global_state info) :: IO (IORef (Maybe Int64))
start <- atomicModifyIORef' state (\value -> (Nothing, value))
chunk <- R.output (c_duckdb_v2_copy_from_exec_get_output_chunk info)
vector <- R.output (c_duckdb_v2_data_chunk_get_vector chunk 0)
writeIntegers vector (maybe [] (\first -> [first .. first + 2]) start)
-- DuckDB calls this only to display progress, so the test checks
-- registration and does not require a call.
progress info _ = R.checked (c_duckdb_v2_copy_from_progress_set_progress info 0.5)
callbacks =
(V2Copy.copyFromCallbacks bind executeRows)
{ V2Copy.copyFromInitGlobalCallback = Just initializeGlobal
, V2Copy.copyFromInitLocalCallback = Just initializeLocal
, V2Copy.copyFromProgressCallback = Just progress
}
V2Copy.registerCopyFromCallbacks connection "managed_input" callbacks
void (execute_ connection "CREATE TABLE imported (value BIGINT)")
void (execute_ connection "COPY imported FROM 'unused' (FORMAT managed_input)")
query_ connection "SELECT value FROM imported ORDER BY value" >>= (@?= map Only ([7, 8, 9] :: [Int64]))
called <- readIORef phases
forM_ ["bind", "global", "local", "exec"] \phase -> assertBool (Text.unpack phase) (phase `elem` called)
-- | Contain exceptions from native COPY callbacks.
testCopyFailure :: Assertion
testCopyFailure = withConnection ":memory:" \connection -> do
V2Copy.registerCopyFromCallbacks
connection
"managed_failure"
(V2Copy.copyFromCallbacks (\_ _ -> pure ()) (\_ _ -> ioError (userError "managed COPY failure")))
void (execute_ connection "CREATE TABLE failed_import (value BIGINT)")
result <- try (execute_ connection "COPY failed_import FROM 'unused' (FORMAT managed_failure)")
case result of
Left SQLError{sqlErrorMessage} -> assertBool "callback diagnostic" ("managed COPY failure" `Text.isInfixOf` sqlErrorMessage)
Right _ -> assertFailure "expected callback failure"
query_ connection "SELECT 42" >>= (@?= [Only (42 :: Int64)])
-- | Wrap a managed one-file table reader with native file discovery.
testMultiFile :: Assertion
testMultiFile = withDirectory \directory -> withConnection ":memory:" \connection -> do
let first = directory </> "first.mock"
second = directory </> "second.mock"
writeFile first "7"
writeFile second "11"
bindInfoCalls <- registerSingleFileReader connection
Copy.registerMultiFileFunction connection "managed_files" "managed_single" (Copy.MultiFileOptions (Just "managed fixture") (Just "mock"))
query connection "SELECT value FROM managed_files(?) ORDER BY value" (Only (Text.pack (directory </> "*.mock")))
>>= (@?= map Only ([7, 11] :: [Int64]))
let paths = [Text.pack first, Text.pack second]
query connection "SELECT value, filename FROM managed_files(?::VARCHAR[], filename=true) ORDER BY value" (Only paths)
>>= (@?= [(7 :: Int64, Text.pack first), (11, Text.pack second)])
forM_ [("left", 3), ("right", 5 :: Int)] \(part, value) -> do
let path = directory </> ("part=" <> part)
createDirectory path
writeFile (path </> "data.mock") (show value)
query
connection
"SELECT value,part FROM managed_files(?,hive_partitioning=true,union_by_name=true) ORDER BY value"
(Only (Text.pack (directory </> "part=*" </> "*.mock")))
>>= (@?= [(3 :: Int64, "left" :: Text), (5, "right")])
readIORef bindInfoCalls >>= assertBool "table get-bind-info callback" . (> 0)
-- | Register a table function that returns the integer stored in one text file.
registerSingleFileReader :: Connection -> IO (IORef Int)
registerSingleFileReader connection = do
calls <- newIORef 0
R.registerTableFunction
connection
"managed_single"
[R.parameter "path" (R.TypeExpression "VARCHAR")]
(R.tableCallbacks bind executeRows){R.tableInitGlobalCallback = Just initialize, R.tableGetBindInfoCallback = Just (getBindInfo calls)}
pure calls
where
bind input result context = do
path <- bracket (R.output (c_duckdb_v2_function_bind_get_arg_value input 0)) (R.destroy c_duckdb_v2_value_destroy) \value -> R.output (c_duckdb_v2_value_get_varchar value) >>= R.readText
value <- read <$> readFile (Text.unpack path) :: IO Int64
R.setCallbackState (c_duckdb_v2_function_bind_set_bind_data input) value
bracket
( R.output (c_duckdb_v2_context_get_factory context) >>= \factory -> R.withText "BIGINT" (R.output . c_duckdb_v2_factory_create_type_from_text factory)
)
(R.destroy c_duckdb_v2_logical_type_destroy)
\dtype ->
R.withIdentifier "value" \name -> R.checked (c_duckdb_v2_table_function_bind_add_result_column result name dtype)
getBindInfo calls info context = do
modifyIORef' calls (+ 1)
value <- R.getCallbackState (c_duckdb_v2_table_function_get_bind_info_get_bind_data info) :: IO Int64
factory <- R.output (c_duckdb_v2_context_get_factory context)
bracket (R.withText "value" (R.output . c_duckdb_v2_value_create_varchar factory)) (R.destroy c_duckdb_v2_value_destroy) $ \identifier ->
R.checked (c_duckdb_v2_table_function_get_bind_info_set_column_identifier info 0 (ConstPtr nullPtr) 0 identifier)
bracket (R.output (c_duckdb_v2_value_create_bigint factory value)) (R.destroy c_duckdb_v2_value_destroy) $ \native ->
R.withText "fixture_value" $ \(ConstPtr key) -> R.checked (c_duckdb_v2_table_function_get_bind_info_set_option info key native)
initialize info _ = do
value <- R.getCallbackState (c_duckdb_v2_table_function_init_global_get_bind_data info)
state <- newIORef (Just (value :: Int64))
R.setCallbackState (c_duckdb_v2_table_function_init_global_set_global_state info) state
executeRows info _ = do
state <- R.getCallbackState (c_duckdb_v2_table_function_exec_get_global_state info) :: IO (IORef (Maybe Int64))
value <- atomicModifyIORef' state (\previous -> (Nothing, previous))
chunk <- R.output (c_duckdb_v2_table_function_exec_get_output_chunk info)
vector <- R.output (c_duckdb_v2_data_chunk_get_vector chunk 0)
writeIntegers vector (maybe [] pure value)
-- | Read a nonnegative integer from native statistics.
number :: FieldValue -> Integer
number (FieldWord64 value) = toInteger value
number (FieldInt64 value) = toInteger value
number value = error ("unexpected statistics value " <> show value)
-- | Create and remove a unique directory for native file operations.
withDirectory :: (FilePath -> IO a) -> IO a
withDirectory action = bracket create removePathForcibly action
where
create = do
root <- getTemporaryDirectory
(path, handle) <- openTempFile root "duckdb-managed-copy"
hClose handle
removeFile path
createDirectory path
pure path