duckdb-simple-1.0.0.0: src/Database/DuckDB/Simple/Materialize.hs
{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE TypeApplications #-}
-- |
-- Module : Database.DuckDB.Simple.Materialize
-- Description : Readers that copy native vectors and values to field values.
module Database.DuckDB.Simple.Materialize (
VectorReader,
FallbackReaders (..),
rejectRows,
prepareVariantReader,
readCallbackVariant,
readInstanceVariant,
createHelperInstance,
ReadFactory (..),
callbackReadFactory,
readFactoryHandle,
prepareGeometryReader,
materializeValue,
prepareVectorReader,
TypeTree,
EnumLabels,
prepareVectorReaderWith,
) where
import Control.Exception (bracket)
import Control.Monad (foldM, forM, when)
import Data.Array (Array, bounds, elems, listArray, rangeSize, (!))
import Data.IORef (IORef, modifyIORef', newIORef, readIORef, writeIORef)
import Data.Int (Int16, Int32, Int64)
import qualified Data.IntMap.Strict as IntMap
import qualified Data.Map.Strict as Map
import Data.Text (Text)
import qualified Data.Text as Text
import Data.Void (Void)
import Data.Word (Word16, Word32, Word64, Word8)
import Database.DuckDB.Simple.Element (
chunkIsRowValid,
decodeElement,
duckDBHugeIntToInteger,
vectorElementType,
withVectorType,
)
import Database.DuckDB.Simple.FromField (
DecimalValue (..),
FieldValue (..),
RawGeometry (..),
)
import Database.DuckDB.Simple.LogicalRep (
LogicalTypeRep (..),
StructField (..),
StructValue (..),
TupleValue (..),
UnionMemberType (..),
UnionValue (..),
aliasBase,
logicalTypeSQL,
)
import Database.DuckDB.Simple.Native
import Database.DuckDB.Simple.V2.LogicalRep (logicalTypeToBaseRep, logicalTypeToRep)
import qualified Database.DuckDB.Simple.VariantCodec as VariantCodec
import Foreign.C.ConstPtr (ConstPtr (..))
import Foreign.Marshal.Alloc (alloca)
import Foreign.Ptr (Ptr, castPtr, nullPtr)
import Foreign.Storable (Storable (..), peekElemOff)
import GHC.Records (getField)
-- | Prepare a reader for the rows of a vector. The reader must not outlive the vector.
type VectorReader = DuckDBV2VectorHandle -> IO (Int -> IO FieldValue)
-- | Readers that need state outside the vector. Type metadata needs a factory,
-- VARIANT needs a helper connection, and GEOMETRY needs a factory for casts.
-- See Note [Type trees]. For VARIANT, see Note [VARIANT decoding] in
-- "Database.DuckDB.Simple.VariantCodec".
data FallbackReaders = FallbackReaders
{ typeReader :: DuckDBV2LogicalTypeHandle -> IO LogicalTypeRep
, variantReader :: VectorReader
, geometryReader :: VectorReader
}
-- | A reader for a type that the caller cannot read. It reads each row when the
-- row is read, not when the vector is prepared. A NULL row gives 'FieldNull',
-- and a row that holds a value raises the error. Thus a value of a nested type
-- with no value of that type, such as an empty VARIANT list or a NULL STRUCT,
-- gives its value. The header commits to no view layout for VARIANT and
-- GEOMETRY, so the reader reads each row with @duckdb_v2_vector_get_value@,
-- whose row is logical. 'prepareVectorReaderWith' flattens the vector, so the
-- row that the reader gets is the logical row. See Note [Vector flatten]. The
-- reader creates and destroys one native value for each row.
rejectRows :: IO FieldValue -> VectorReader
rejectRows raise vector = pure \row ->
bracket (output (c_duckdb_v2_vector_get_value vector (fromIntegral row))) destroyValue \value -> do
isNull <- output (c_duckdb_v2_value_is_null value)
if isNull /= 0 then pure FieldNull else raise
-- | Copy VARIANT rows before releasing the borrowed helper connection. The
-- state belongs to that connection. The decoder reads logical rows. See
-- Note [Vector flatten] and Note [VARIANT decoding] in
-- "Database.DuckDB.Simple.VariantCodec".
prepareVariantReader :: VariantCodec.VariantState -> VectorReader
prepareVariantReader =
VariantCodec.prepareVariantDecoder
(materializeValue noVariant . InstanceFactory)
( \factory vector ->
prepareVectorReader (FallbackReaders (logicalTypeToRep factory) noVariant (prepareGeometryReader (InstanceFactory factory))) vector
>>= ($ 0)
)
where
noVariant = rejectRows (throwInternalError (Text.pack "expected a value without VARIANT"))
{- Note [VARIANT reads inside callbacks]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The VARIANT decoder runs SQL. A callback cannot run it on the executing
connection. Even a new connection to the host instance can need a lock that
the caller holds, for example during SET GLOBAL. An extension has no host
instance handle at all.
The decoder uses an instance with only built-in types, casts and system
functions. A scalar or aggregate registration opens its instance on demand.
Each vector uses a separate connection on that instance. Other typed
callbacks, including extension COPY TO, open a private instance for each
VARIANT vector. The decoder copies the rows before it closes the connection.
See Note [Function argument readers] in "Database.DuckDB.Simple.Typed"
and Note [VARIANT decoding] in "Database.DuckDB.Simple.VariantCodec".
-}
-- | Decode a callback's VARIANT vector independently of the invoking database.
readCallbackVariant :: VectorReader
readCallbackVariant vector =
bracket (output c_duckdb_v2_environment_create) (destroy c_duckdb_v2_environment_destroy) \env ->
bracket (createHelperInstance env) (destroy c_duckdb_v2_instance_destroy) (`readInstanceVariant` vector)
-- | Copy a VARIANT vector through a temporary connection on the supplied instance.
readInstanceVariant :: DuckDBV2InstanceHandle -> VectorReader
readInstanceVariant instance_ vector =
bracket (output (c_duckdb_v2_connection_create instance_)) (destroy c_duckdb_v2_connection_destroy) \helper ->
withInstanceFactory instance_ \factory ->
bracket (VariantCodec.newVariantState helper factory) VariantCodec.destroyVariantState (`prepareVariantReader` vector)
-- | Create an instance for synchronous metadata work, without background workers.
createHelperInstance :: DuckDBV2EnvironmentHandle -> IO DuckDBV2InstanceHandle
createHelperInstance env =
withTextArray (map Text.pack ["threads", "async_threads"]) \names ->
withTextArray (map Text.pack ["1", "0"]) \values ->
output (c_duckdb_v2_instance_create_with_options env (ConstPtr names) (ConstPtr values) 2)
-- | The factory of a read, and its kind. The kind decides how
-- 'prepareGeometryReader' casts GEOMETRY to BLOB. See
-- Note [User casts and the casts of the library].
data ReadFactory
= -- | The factory of an instance. Its casts are the built-in casts.
InstanceFactory DuckDBV2FactoryHandle
| -- | The factory of a callback context. Its casts include the casts that a
-- user registers.
ContextFactory DuckDBV2FactoryHandle
-- | The factory of a read in a callback: the factory of the instance from
-- 'Database.DuckDB.Simple.Callback.targetCastFactory', or the factory of the
-- context for an extension target, which has no instance.
callbackReadFactory :: Maybe DuckDBV2FactoryHandle -> DuckDBV2FactoryHandle -> ReadFactory
callbackReadFactory casts context = maybe (ContextFactory context) InstanceFactory casts
-- | The handle of a read factory.
readFactoryHandle :: ReadFactory -> DuckDBV2FactoryHandle
readFactoryHandle (InstanceFactory factory) = factory
readFactoryHandle (ContextFactory factory) = factory
-- | Copy one value through an owned vector. The caller retains the factory owner.
-- The first argument reads VARIANT. 'prepareGeometryReader' reads GEOMETRY with
-- the factory.
materializeValue :: VectorReader -> ReadFactory -> DuckDBV2ValueHandle -> IO FieldValue
materializeValue readVariant factory value =
bracket (output (c_duckdb_v2_value_get_logical_type value)) (destroy c_duckdb_v2_logical_type_destroy) \logical ->
withOwnedVector (readFactoryHandle factory) logical \vector -> do
checked (c_duckdb_v2_vector_set_size vector 1)
checked (c_duckdb_v2_vector_set_value vector 0 value)
prepareVectorReader (FallbackReaders (logicalTypeToRep (readFactoryHandle factory)) readVariant (prepareGeometryReader factory)) vector
>>= ($ 0)
{- Note [Type trees]
~~~~~~~~~~~~~~~~~~~~
A reader of a vector uses two kinds of type information.
- A layout parameter gives the layout of the bytes, for example the width of a
DECIMAL, the size of an ARRAY, or the size of an ENUM dictionary. A decoder
takes each layout parameter from the type of its own vector, because that
type gives the layout of the bytes. The header of duckdb_v2_vector_get_view
says that the data has the layout that the logical type of the vector
implies.
- A description is the field names and field types of a STRUCT or TUPLE, or
the members of a UNION. A decoder takes a description from a type tree: the
LogicalTypeRep that an action of type TypeTree gives. A decoder runs the
action only when it needs a description.
Thus a wrong tree gives an error or a wrong description, but it does not change
how bytes are read. checkChildCount raises an error when a tree gives another
child count than the vector has.
These are the sources of a tree.
- prepareVectorReader converts the type of the vector.
- A caller of prepareVectorReaderWith gives the action. A result converts the
tree of a column from the vector of the first chunk that needs it, and keeps
the tree for one execution (prepareChunkReaders).
- A nested decoder gives each child an action that takes the tree of the child
from its own tree (elementTree, prepareStructDecoder, prepareTupleDecoder, and
prepareUnionDecoder). The action of a child runs the action of the parent
only when the child needs its tree.
- The MAP decoder gives its key reader and its value reader no tree, and the
UNION decoder gives its tag reader no tree. Each of these readers converts
the type of its own vector when it needs a description.
The readers do not use the result schema. The header of
duckdb_v2_result_get_schema calls the schema the authoritative source of the
column types, and says that vectors do not carry their own type. But the header
of duckdb_v2_vector_get_logical_type says that a vector has the type that was
set on creation, and the layout follows the type of the vector. So the vector
type is the source of each type.
-}
-- | An action that gives the type tree of one vector. A decoder runs it only
-- when it needs a description. See Note [Type trees].
type TypeTree = IO LogicalTypeRep
-- | Read the type, data, and validity of a vector, and prepare its reader. A
-- decoder that needs the type tree converts the type of the vector. See
-- Note [Type trees].
prepareVectorReader :: FallbackReaders -> DuckDBV2VectorHandle -> IO (Int -> IO FieldValue)
prepareVectorReader readers vector = do
-- The cache of ENUM labels lives as long as this reader. See Note [ENUM labels].
labels <- newIORef IntMap.empty
-- See Note [Type aliases] in "Database.DuckDB.Simple.LogicalRep".
prepareVectorReaderWith readers (withVectorType vector (typeReader readers)) (Just labels) vector
-- | The labels of an ENUM type that readers read, by index. 'Nothing' means
-- that the type tree holds the labels. See Note [ENUM labels].
type EnumLabels = Maybe (IORef (IntMap.IntMap Text))
{- Note [Vector flatten]
~~~~~~~~~~~~~~~~~~~~~~~~
vector_flatten changes a vector to the FLAT representation in place. A reader
flattens each vector before it reads the view. A writer of a callback output
vector flattens the vector before it writes rows.
Read (prepareVectorReaderWith).
- The header of duckdb_v2_vector_get_view says: "Rejects VECTOR_TYPE_OTHER;
call vector_flatten first." The flatten also changes a CONSTANT or DICTIONARY
vector to FLAT.
- In DuckDB 2.0.0 the view of a FLAT vector has no selection
(duckdb_v2_vector_get_view in src/main/capi/v2/capi_v2_vector.cpp). The header
does not say this. Then the physical row is the logical row. The readers of
VARIANT and GEOMETRY need this, because they read logical rows (rejectRows,
prepareVariantReader, and prepareGeometryReader). The reader still applies
the selection of a view that has one.
- The flatten changes the buffers of the vector, so it runs before the view is
read.
- In DuckDB 2.0.0 the flatten is recursive: it also flattens the child of a
LIST or MAP vector and the fields of a STRUCT vector (VectorListBuffer::Flatten
and VectorStructBuffer::Flatten). The header does not say this. The MAP
decoder (prepareMapDecoder) depends on this, because it takes the key and
value vectors from the entry STRUCT and does not flatten that STRUCT itself.
Write. A writer of a callback output vector sets the size of the vector, and
then flattens the vector (writeRows and writeResults). A writer of an
owned vector from withOwnedVector does not flatten (materializeValue, and the
writers of one-row vectors in "Database.DuckDB.Simple.ToField").
- The writer sets the size before the flatten. In DuckDB 2.0.0,
duckdb_v2_vector_flatten flattens the rows of the current size of the vector:
it calls Vector::Flatten without a row count
(src/main/capi/v2/capi_v2_vector.cpp). duckdb_v2_vector_set_size reserves
memory only for a vector that is already FLAT. Thus after this order the
vector is FLAT and has the new number of rows. The header gives no order.
- A new vector of a data chunk is FLAT. The header of
duckdb_v2_data_chunk_create says: "Allocates one FLAT vector per element of
the types array". Thus the writer of an owned vector needs no flatten. In
DuckDB 2.0.0 the children of such a vector are FLAT too: the vector cache
resets each child of a LIST, ARRAY, or STRUCT vector to FLAT
(VectorCacheEntry::ResetFromCache in src/common/types/vector_cache.cpp). The
header does not say this for the children. The writer of a UNION value writes
the child vectors of a new vector (unionValueDuckValue in
"Database.DuckDB.Simple.ToField").
- The set operations check the row against the size. The header of
duckdb_v2_vector_set_value and duckdb_v2_vector_set_null says that each call
requires a FLAT vector and checks the row against vector_get_size.
- writeResults reads the data pointer and the validity pointer after the
flatten. The header of duckdb_v2_vector_get_data_mutable says that the storage
shape of the vector must not change while the pointer is in use.
- prepareResultWriter does not change the size and does not flatten. It raises
an error for a vector that is not FLAT.
-}
-- | Read the type, data, and validity of a vector, and prepare its reader. The
-- action gives the type tree of the vector. A nested decoder gives each child
-- an action that takes the tree of the child from this tree. The function
-- flattens the vector first. See Note [Vector flatten] and Note [Type trees].
prepareVectorReaderWith :: FallbackReaders -> TypeTree -> EnumLabels -> DuckDBV2VectorHandle -> IO (Int -> IO FieldValue)
prepareVectorReaderWith readers tree labels vector = do
-- Flatten before the view is read. See Note [Vector flatten].
checked (c_duckdb_v2_vector_flatten vector)
dtype <- vectorElementType vector
view <- output (c_duckdb_v2_vector_get_view vector)
let ConstPtr dataPtr = getField @"data'" view
ConstPtr validity = getField @"validity" view
ConstPtr selection = getField @"sel" view
-- See Note [Type aliases] in "Database.DuckDB.Simple.LogicalRep".
readValue <- prepareValueReader readers (aliasBase <$> tree) labels dtype vector dataPtr validity
pure \row -> do
when (row < 0 || fromIntegral row >= getField @"count" view) $
throwInternalError (Text.pack "vector row index out of range")
physical <- if selection == nullPtr then pure row else fromIntegral <$> peekElemOff selection row
readValue physical
{- Note [ENUM labels]
~~~~~~~~~~~~~~~~~~~~
A FieldEnum holds the index and the label of an ENUM value. The label is the
text of the cast of DuckDB to VARCHAR. The reader does not cast the vector, so
a user cast cannot change a label. The reader takes the label from one of two
sources:
- A cache (EnumLabels). The reader reads the label of an index through the
dictionary of the vector type (duckdb_v2_logical_type_get_param) at the first
row that has the index, and keeps it. Thus a query reads only the labels of
the values that it reads.
- The type tree. A STRUCT, TUPLE, or UNION decoder converts the type of the
value, which reads the whole dictionary of each ENUM type in it (see
logicalTypeToRep). The reader of a child uses the labels of that tree, and it
raises an error when the tree has another dictionary size than the vector.
A result column has one cache, which lives for one execution, as the tree of
the column (statementStreamColumnLabels in "Database.DuckDB.Simple.Internal").
The column reader gives the cache to the element reader of a LIST or an ARRAY,
because one column has at most one ENUM type outside a STRUCT, a TUPLE, a
UNION, and a MAP. prepareVectorReader makes a new cache for each reader. These
readers use it: the key and value readers of a MAP, the tag reader of a UNION,
the readers of typed arguments, function arguments, and COPY TO rows, and
materializeValue. There the cache lives for one vector.
Each index is checked against the parameter count of the vector type. An index
out of range raises an error, so the reader cannot give a wrong label.
-}
-- | Prepare metadata once for a vector. The reader must not outlive its chunk.
prepareValueReader
:: FallbackReaders
-> TypeTree
-> EnumLabels
-> DuckDBV2LogicalTypeId
-> DuckDBV2VectorHandle
-> Ptr Void
-> Ptr Word64
-> IO (Int -> IO FieldValue)
prepareValueReader readers tree labels dtype vector dataPtr validity = case dtype of
DuckDBV2LogicalTypeIdVariant -> variantReader readers vector
DuckDBV2LogicalTypeIdGeometry -> geometryReader readers vector
DuckDBV2LogicalTypeIdArray -> whenValid FieldArray <$> prepareArrayDecoder readers tree labels vector
DuckDBV2LogicalTypeIdList -> whenValid FieldList <$> prepareListDecoder readers tree labels vector dataPtr
DuckDBV2LogicalTypeIdMap -> whenValid FieldMap <$> prepareMapDecoder readers vector dataPtr
DuckDBV2LogicalTypeIdStruct -> whenValid FieldStruct <$> prepareStructDecoder readers tree vector
DuckDBV2LogicalTypeIdTuple -> whenValid FieldTuple <$> prepareTupleDecoder readers tree vector
DuckDBV2LogicalTypeIdUnion -> whenValid FieldUnion <$> prepareUnionDecoder readers tree vector
DuckDBV2LogicalTypeIdDecimal -> do
-- See Note [Type aliases] in "Database.DuckDB.Simple.LogicalRep".
(width, scale) <-
withVectorType vector logicalTypeToBaseRep >>= \case
LogicalTypeDecimal width scale -> pure (width, scale)
_ -> throwInternalError (Text.pack "invalid DECIMAL type")
pure (whenValid (FieldDecimal . DecimalValue width scale) (decimalStorage dataPtr width))
DuckDBV2LogicalTypeIdEnum -> do
-- See Note [ENUM labels].
count <- withVectorType vector (output . c_duckdb_v2_logical_type_get_param_count)
label <- case labels of
Just cache -> pure (cachedLabel cache)
Nothing ->
tree >>= \case
LogicalTypeEnum entries | toInteger (rangeSize (bounds entries)) == toInteger count -> pure (pure . (entries !) . fromIntegral)
_ -> throwInternalError (Text.pack "invalid ENUM type")
let readEnum row = do
index <- enumStorage dataPtr count row
when (toInteger index >= toInteger count) $ throwInternalError (Text.pack "ENUM index out of range")
FieldEnum index <$> label index
pure (whenValid id readEnum)
DuckDBV2LogicalTypeIdSQLNull -> pure \_ -> pure FieldNull
_ -> pure (whenValid id (decodeElement dtype dataPtr))
where
-- Read a label through the dictionary of the vector type at the first row
-- that needs it. See Note [ENUM labels].
cachedLabel cache index =
IntMap.lookup (fromIntegral index) <$> readIORef cache >>= \case
Just text -> pure text
Nothing -> do
text <- withVectorType vector \logical -> alloca \nameSlot ->
bracket (output (c_duckdb_v2_logical_type_get_param logical (fromIntegral index) nameSlot)) destroyValue \value ->
output (c_duckdb_v2_value_get_varchar value) >>= readText
modifyIORef' cache (IntMap.insert (fromIntegral index) text)
pure text
-- See Note [Vector memory layouts] in "Database.DuckDB.Simple.Element".
whenValid wrap decode row = do
valid <- chunkIsRowValid validity (fromIntegral row)
if valid then wrap <$> decode row else pure FieldNull
{- Note [User casts and the casts of the library]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A cast that a user registers is a cast function of each connection of the
database, also of the helper connection of the VARIANT codec. DuckDB uses a
registered cast in place of the built-in cast for the same pair of types. The
header says: "A connection's or callback's factory casts with that client's
cast functions and settings". It also says that the factory of an instance
"supports only the built-in casts, with default settings".
The library runs casts of DuckDB for its own work. A user cast does not change
their result:
- Reads. prepareGeometryReader casts a GEOMETRY vector to BLOB. A result, the
value of an option (getConfigValue), and the default value of an option
(getConfigDefaultValue) read a GEOMETRY value with the factory of the
instance, and so does queryMetrics. Callbacks registered on a connection
use the same instance factory.
A scalar or aggregate function registered by a loadable extension uses
its private instance factory. See Note [Function argument readers] in
"Database.DuckDB.Simple.Typed". Both paths cast the whole vector.
Other extension callbacks have a context factory, which includes user
casts. For those callbacks, withDefaultCastVector writes each row to a
BLOB vector with the default cast ("Database.DuckDB.Simple.Native").
This prevents a user cast from GEOMETRY to BLOB from calling itself when
its callback reads a GEOMETRY argument.
TODO: When V2 exposes WKB directly, copy it from the input vector and
remove the intermediate BLOB cast.
- withObjectChildren and withCast cast VARIANT values to the types of their
payloads ("Database.DuckDB.Simple.VariantCodec"). The codec also reads a
GEOMETRY payload. It uses the factory of the instance of its helper
connection (variantCasts).
- Values. rawGeometryDuckValue casts BLOB to GEOMETRY, the ToDuckValue instance
of Geometry casts VARCHAR to GEOMETRY, enumDuckValue casts VARCHAR to ENUM,
and variantDuckValue casts its payload to VARIANT
("Database.DuckDB.Simple.ToField"). defaultCast writes each value to a
one-row vector of the target type and reads it back. vector_set_value uses
"the non-strict default cast". This is the documented cell writer for
GEOMETRY and VARIANT, which have no committed view layout. For ENUM, it avoids
value_cast, which includes registered casts. A VARCHAR-to-ENUM user cast
must not change the label of a FieldEnum. The factory resolves the target
type. Only a connection or callback factory can construct a GEOMETRY
type with a CRS. See Note [Raw GEOMETRY parameters] in
"Database.DuckDB.Simple.ToField". UNION values use a vector to preserve the
explicit member tag. See Note [UNION values] in that module.
- The bind-time CRS query passes a BLOB literal to ST_GeomFromWKB and binds
the label as VARCHAR. It needs no user cast. See Note [Bind-time CRS types] in
"Database.DuckDB.Simple.V2.LogicalRep".
These casts still use the cast functions of the session, so a user cast for
the same pair changes their result:
- setConfigValue and setConfigOption. DuckDB casts the value to the type of the
option as SQL SET does.
- The casts of the SQL text. DuckDB runs them with the casts of the session.
The library does not choose them.
-}
-- | Copy the WKB bytes and CRS of every GEOMETRY row in a vector. GEOMETRY has
-- no committed vector layout, so a cast to BLOB reads the rows. The reader takes
-- logical rows. See Note [Vector flatten]. With the factory of an instance, one
-- native cast casts the vector. With the factory of a context, the default cast
-- of 'withDefaultCastVector' casts each row. See
-- Note [User casts and the casts of the library].
prepareGeometryReader :: ReadFactory -> DuckDBV2VectorHandle -> IO (Int -> IO FieldValue)
prepareGeometryReader readFactory vector = do
-- See Note [Type aliases] in "Database.DuckDB.Simple.LogicalRep".
crs <-
withVectorType vector logicalTypeToBaseRep >>= \case
LogicalTypeGeometry value -> pure value
_ -> throwInternalError (Text.pack "invalid GEOMETRY type")
size <- output (c_duckdb_v2_vector_get_size vector)
let count = fromIntegral size
factory = readFactoryHandle readFactory
readBlobs target = do
let notBlob = rejectRows (throwInternalError (Text.pack "expected a BLOB vector"))
readBlob <- prepareVectorReader (FallbackReaders (logicalTypeToRep factory) notBlob notBlob) target
forM [0 .. count - 1] readBlob
blobs <-
bracket
(output (c_duckdb_v2_factory_create_type_from_id factory DuckDBV2LogicalTypeIdBlob (ConstPtr nullPtr) (ConstPtr nullPtr) 0))
destroyLogicalType
\blob -> case readFactory of
InstanceFactory _ -> withOwnedVector factory blob \target -> do
checked (c_duckdb_v2_vector_cast factory vector target DuckDBV2CastModeNormal nullPtr nullPtr)
readBlobs target
ContextFactory _ ->
withDefaultCastVector
factory
blob
size
(\row write -> bracket (output (c_duckdb_v2_vector_get_value vector row)) destroyValue write)
readBlobs
let rows = listArray (0, count - 1) blobs :: Array Int FieldValue
pure \row -> case rows ! row of
FieldBlob bytes -> pure (FieldGeometry (RawGeometry bytes crs))
FieldNull -> pure FieldNull
_ -> throwInternalError (Text.pack "GEOMETRY cast did not produce a BLOB")
-- | Read a DECIMAL. DuckDB stores it in the smallest integer type for its width.
-- See Note [Vector memory layouts] in "Database.DuckDB.Simple.Element".
decimalStorage :: Ptr Void -> Word8 -> Int -> IO Integer
decimalStorage dataPtr width rowIdx
| width <= 4 = toInteger <$> peekElemOff (castPtr dataPtr :: Ptr Int16) rowIdx
| width <= 9 = toInteger <$> peekElemOff (castPtr dataPtr :: Ptr Int32) rowIdx
| width <= 18 = toInteger <$> peekElemOff (castPtr dataPtr :: Ptr Int64) rowIdx
| otherwise = duckDBHugeIntToInteger <$> peekElemOff (castPtr dataPtr :: Ptr DuckDBV2HugeintT) rowIdx
-- | Read an ENUM index. DuckDB stores it in the smallest unsigned type for the dictionary size.
-- See Note [Vector memory layouts] in "Database.DuckDB.Simple.Element".
enumStorage :: Ptr Void -> DuckDBIdx -> Int -> IO Word32
enumStorage dataPtr count rowIdx
| count <= 255 = fromIntegral <$> peekElemOff (castPtr dataPtr :: Ptr Word8) rowIdx
| count <= 65535 = fromIntegral <$> peekElemOff (castPtr dataPtr :: Ptr Word16) rowIdx
| otherwise = peekElemOff (castPtr dataPtr :: Ptr Word32) rowIdx
-- | Read the ARRAY size and prepare the element reader once for a vector. The
-- size gives the layout of the elements, so it comes from the type of this
-- vector. Parameter 1 of an ARRAY type is the size, as 'logicalTypeToRep' reads
-- it. The decoder does not convert the element type. The element reader takes
-- its tree from the tree of the ARRAY. See Note [Type trees].
-- See Note [Constant stack at safe calls] in "Database.DuckDB.Simple.Native".
prepareArrayDecoder :: FallbackReaders -> TypeTree -> EnumLabels -> DuckDBV2VectorHandle -> IO (Int -> IO (Array Int FieldValue))
prepareArrayDecoder readers tree labels vector = do
size <- withVectorType vector \logical -> alloca \name ->
bracket (output (c_duckdb_v2_logical_type_get_param logical 1 name)) destroyValue (output . c_duckdb_v2_value_get_bigint)
arraySize <- ensureWithinIntRange (Text.pack "array size") (fromIntegral size)
readChild <- vectorChild vector 0 >>= prepareVectorReaderWith readers (elementTree tree) labels
pure \rowIdx -> do
let baseIdx = rowIdx * arraySize
values <- reverse <$> foldM (\values delta -> (: values) <$> readChild (baseIdx + delta)) [] [0 .. arraySize - 1]
pure (listArray (0, arraySize - 1) values)
-- | Prepare the element reader once for a LIST vector. The element reader
-- takes its tree from the tree of the LIST. See Note [Type trees].
-- See Note [Constant stack at safe calls] in "Database.DuckDB.Simple.Native".
prepareListDecoder :: FallbackReaders -> TypeTree -> EnumLabels -> DuckDBV2VectorHandle -> Ptr Void -> IO (Int -> IO [FieldValue])
prepareListDecoder readers tree labels vector dataPtr = do
readChild <- vectorChild vector 0 >>= prepareVectorReaderWith readers (elementTree tree) labels
pure \rowIdx -> do
entry <- peekElemOff (castPtr dataPtr :: Ptr DuckDBV2ListEntry) rowIdx
(baseIdx, len) <- listEntryBounds (Text.pack "list") entry
reverse <$> foldM (\values delta -> (: values) <$> readChild (baseIdx + delta)) [] [0 .. len - 1]
-- | Prepare the key reader and the value reader once for a MAP vector. The
-- decoder gives them no type tree. Each converts the type of its own vector when
-- it needs a description. A tree of the MAP holds the key type and the value
-- type together. A value reader that took its tree from it would thus also
-- convert the key type, for example the dictionary of a large ENUM. The
-- decoder does not flatten the entry STRUCT. See Note [Vector flatten] and
-- Note [Type trees].
-- See Note [Constant stack at safe calls] in "Database.DuckDB.Simple.Native".
prepareMapDecoder :: FallbackReaders -> DuckDBV2VectorHandle -> Ptr Void -> IO (Int -> IO [(FieldValue, FieldValue)])
prepareMapDecoder readers vector dataPtr = do
entries <- vectorChild vector 0
readKey <- vectorChild entries 0 >>= prepareVectorReader readers
readValue <- vectorChild entries 1 >>= prepareVectorReader readers
pure \rowIdx -> do
entry <- peekElemOff (castPtr dataPtr :: Ptr DuckDBV2ListEntry) rowIdx
(baseIdx, len) <- listEntryBounds (Text.pack "map") entry
reverse
<$> foldM
( \pairs delta -> do
let childIdx = baseIdx + delta
keyValue <- readKey childIdx
valueValue <- readValue childIdx
pure ((keyValue, valueValue) : pairs)
)
[]
[0 .. len - 1]
-- | Read the STRUCT type and prepare a reader for each child once for a vector.
-- The decoder reads a valid row. It must not outlive its chunk. See
-- Note [Type trees].
prepareStructDecoder :: FallbackReaders -> TypeTree -> DuckDBV2VectorHandle -> IO (Int -> IO (StructValue FieldValue))
prepareStructDecoder readers tree vector = do
structFields <-
tree >>= \case
LogicalTypeStruct typeArray -> pure typeArray
other ->
throwInternalError (Text.pack "expected STRUCT logical type, but saw " <> logicalTypeSQL other)
let typeList = elems structFields
count = length typeList
indexMap =
Map.fromList (zip (map structFieldName typeList) [0 ..])
checkChildCount "STRUCT" vector count
childReaders <-
forM (zip [0 ..] typeList) \(childIdx, StructField{structFieldName, structFieldValue}) -> do
readChild <- vectorChild vector childIdx >>= prepareVectorReaderWith readers (pure structFieldValue) Nothing
pure (structFieldName, readChild)
pure \rowIdx -> do
valueFields <-
forM childReaders \(name, readChild) -> do
value <- readChild rowIdx
pure StructField{structFieldName = name, structFieldValue = value}
let fieldArray =
listArray (0, count - 1) valueFields
pure
StructValue
{ structValueFields = fieldArray
, structValueTypes = structFields
, structValueIndex = indexMap
}
-- | Read positional field metadata and prepare readers that borrow the tuple.
-- See Note [Type trees].
prepareTupleDecoder :: FallbackReaders -> TypeTree -> DuckDBV2VectorHandle -> IO (Int -> IO (TupleValue FieldValue))
prepareTupleDecoder readers tree vector = do
types <-
tree >>= \case
LogicalTypeTuple fields -> pure fields
_ -> throwInternalError (Text.pack "invalid TUPLE type")
let typeList = elems types
count = length typeList
checkChildCount "TUPLE" vector count
fieldReaders <- forM (zip [0 ..] typeList) \(index, fieldType) ->
vectorChild vector index >>= prepareVectorReaderWith readers (pure fieldType) Nothing
pure \row -> do
fields <- mapM ($ row) fieldReaders
pure (TupleValue (listArray (0, count - 1) fields) types)
-- | Read the UNION type and prepare the tag reader once for a vector. Prepare
-- the reader of a member at the first row that selects the member, so that a
-- member that no row selects makes no native call. The decoder reads a valid
-- row. It must not outlive its chunk. See Note [Type trees].
prepareUnionDecoder :: FallbackReaders -> TypeTree -> DuckDBV2VectorHandle -> IO (Int -> IO (UnionValue FieldValue))
prepareUnionDecoder readers tree vector = do
membersArray <-
tree >>= \case
LogicalTypeUnion members -> pure members
other ->
throwInternalError (Text.pack "expected UNION logical type, but saw " <> logicalTypeSQL other)
let memberCount = rangeSize (bounds membersArray)
-- Child 0 of a UNION vector is the tag, and member j is child j + 1. The tag
-- reader needs no type tree.
checkChildCount "UNION" vector (memberCount + 1)
readTag <- vectorChild vector 0 >>= prepareVectorReader readers
readersByTag <- fmap (listArray (0, memberCount - 1)) $
forM (zip [1 ..] (elems membersArray)) \(childIdx, UnionMemberType{unionMemberName, unionMemberType}) -> do
memberVec <- vectorChild vector childIdx
prepared <- newIORef Nothing
pure \row -> do
cached <- readIORef prepared
readMember <- case cached of
Just reader -> pure reader
Nothing -> do
reader <- prepareVectorReaderWith readers (pure unionMemberType) Nothing memberVec
writeIORef prepared (Just reader)
pure reader
payload <- readMember row
pure (unionMemberName, payload)
pure \rowIdx -> do
memberIdx <- readTag rowIdx >>= unionTagIndex
when (memberIdx < 0 || memberIdx >= memberCount) $
throwInternalError (Text.pack "union tag out of range")
(name, payload) <- (readersByTag ! memberIdx) rowIdx
pure
UnionValue
{ unionValueIndex = fromIntegral memberIdx
, unionValueLabel = name
, unionValuePayload = payload
, unionValueMembers = membersArray
}
-- | Convert a decoded UNION tag to a member index.
unionTagIndex :: FieldValue -> IO Int
unionTagIndex = \case
FieldWord8 tagWord -> pure (fromIntegral tagWord :: Int)
FieldWord16 tagWord -> pure (fromIntegral tagWord :: Int)
FieldWord32 tagWord ->
if tagWord <= fromIntegral (maxBound :: Word16)
then pure (fromIntegral tagWord)
else throwInternalError (Text.pack "union tag exceeds Word16 range")
FieldWord64 tagWord ->
if tagWord <= fromIntegral (maxBound :: Word16)
then pure (fromIntegral tagWord)
else throwInternalError (Text.pack "union tag exceeds Word16 range")
FieldInt8 tagInt
| tagInt >= 0 -> pure (fromIntegral tagInt)
| otherwise -> throwInternalError (Text.pack "union tag negative")
FieldInt16 tagInt
| tagInt >= 0 -> pure (fromIntegral tagInt)
| otherwise -> throwInternalError (Text.pack "union tag negative")
FieldInt32 tagInt
| tagInt >= 0 && tagInt <= fromIntegral (maxBound :: Word16) -> pure (fromIntegral tagInt)
| tagInt < 0 -> throwInternalError (Text.pack "union tag negative")
| otherwise -> throwInternalError (Text.pack "union tag exceeds Word16 range")
FieldInt64 tagInt
| tagInt >= 0 && tagInt <= fromIntegral (maxBound :: Word16) -> pure (fromIntegral tagInt)
| tagInt < 0 -> throwInternalError (Text.pack "union tag negative")
| otherwise -> throwInternalError (Text.pack "union tag exceeds Word16 range")
FieldNull ->
throwInternalError (Text.pack "encountered NULL union tag")
other ->
throwInternalError (Text.pack ("unexpected union tag value " <> show other))
listEntryBounds :: Text -> DuckDBV2ListEntry -> IO (Int, Int)
listEntryBounds context (DuckDBV2ListEntry offset lenRaw) = do
base <- ensureWithinIntRange (context <> Text.pack " offset") (fromIntegral offset)
len <- ensureWithinIntRange (context <> Text.pack " length") (fromIntegral lenRaw)
let maxInt = toInteger (maxBound :: Int)
upperBound = toInteger base + toInteger len - 1
when (len > 0 && upperBound > maxInt) $
throwInternalError (context <> Text.pack " bounds exceed Int range")
pure (base, len)
ensureWithinIntRange :: Text -> Word64 -> IO Int
ensureWithinIntRange context value =
let actual = toInteger value
limit = toInteger (maxBound :: Int)
in if actual <= limit
then pure (fromInteger actual)
else throwInternalError (context <> Text.pack " exceeds Int range")
-- | Take the element tree from the tree of a LIST or ARRAY. Read the parent
-- tree only when the element reader needs its tree. See
-- Note [Type trees].
elementTree :: TypeTree -> TypeTree
elementTree tree =
tree >>= \case
LogicalTypeList element -> pure element
LogicalTypeArray element _ -> pure element
_ -> throwInternalError (Text.pack "invalid LIST or ARRAY type")
-- | Raise an error when the type tree of a STRUCT, TUPLE, or UNION vector gives
-- another child count than the vector has. The tree can come from the type of a
-- parent vector, and the decoder must not drop a child. See Note [Type trees].
checkChildCount :: String -> DuckDBV2VectorHandle -> Int -> IO ()
checkChildCount kind vector expected = do
actual <- output (c_duckdb_v2_vector_get_child_count vector)
when (toInteger actual /= toInteger expected) $
throwInternalError
(Text.pack ("invalid " <> kind <> " type: the type has " <> show expected <> " children, but the vector has " <> show actual))
-- | Borrow a nested vector from its parent chunk.
vectorChild :: DuckDBV2VectorHandle -> DuckDBIdx -> IO DuckDBV2VectorHandle
vectorChild vector index = output (c_duckdb_v2_vector_get_child vector index)