diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,19 +1,38 @@ -# 1.0.0 -- ? +# 1.1.0 -- 11-07-2026 +## Fixes: +- Fixed `Array(T)` serialization for blocks containing more than one row (tests extended) + ## Features: +- Added support for `Float(32/64)`, `Decimal(32,64,128,256)`, `Int256`, `Date` column types +- Added support for `Int64`, `Float/Double` settings +- Added support for query parameters + +## Breaking changes: +- Reworked the query serialization type class to support query parameters. + + `toQueryPart` no longer quotes 128-bit and 256-bit integer values. Previously this was introduced as part of a fix for serializing values larger than `999999999999999934463`. + + Added a separate `toQueryPartQuoted` method for serializing quoted values. + + Internal wrappers have been updated to use `toQueryPartQuoted` where appropriate, so existing code should continue to work unless you implemented custom wrappers around `toQueryPart`. + +# 1.0.0 -- 14-12-2025 + +## Fixes: +- Fixed unexpected behavior when the number of result columns was different from expected. + A `UserError` exception `UnmatchedColumnsCount` is now raised in such cases (+ added test) +- Query serialization for UInt128 bigger than 999999999999999934463 (+ added test) + +## Features: - Support and CI for GHCs: `9.12.2` - ~50% optimization of select/insert time and alloc (perf test №1) - Added function `command` for statements with no result - Added `Bool`, `DateTime64`, `UInt256`, `Enum8`, `Enum16` types support - Added `Array(T)` partial support (only for primitive types) -- Added partial support for settings passing (watch `addSetting` fuction) +- Added partial support for settings passing (watch `addSetting` function) - Added TLS support (see separate package [ClickHaskell-tls](https://hackage.haskell.org/package/ClickHaskell-tls)) - Improved ClickHaskell typeclass resolution errors in HLS session - -## Fixes: -- Fixed unexpected behavior when the number of result columns was different from expected. - A `UserError` exception `UnmatchedColumnsCount` is now raised in such cases (+ added test) -- Query serialization for UInt128 bigger than 999999999999999934463 (+ added test) ## Breaking changes: - ### select/insert API changes diff --git a/ClickHaskell.cabal b/ClickHaskell.cabal --- a/ClickHaskell.cabal +++ b/ClickHaskell.cabal @@ -2,7 +2,7 @@ Name: ClickHaskell -Version: 1.0.0 +Version: 1.1.0 Author: Kovalev Dmitry Maintainer: Kovalev Dmitry @@ -35,13 +35,26 @@ ClickHaskell ClickHaskell.Protocol Other-Modules: - ClickHaskell.Columns ClickHaskell.Connection ClickHaskell.Protocol.Client ClickHaskell.Protocol.Data ClickHaskell.Protocol.Server ClickHaskell.Protocol.Settings ClickHaskell.Primitive + ClickHaskell.Primitive.Serialization + ClickHaskell.Primitive.TArray + ClickHaskell.Primitive.TBool + ClickHaskell.Primitive.TDate + ClickHaskell.Primitive.TDateTime + ClickHaskell.Primitive.TDecimal + ClickHaskell.Primitive.TEnum + ClickHaskell.Primitive.TFloat + ClickHaskell.Primitive.TInt + ClickHaskell.Primitive.TLowCardinality + ClickHaskell.Primitive.TNullable + ClickHaskell.Primitive.TString + ClickHaskell.Primitive.TUInt + ClickHaskell.Primitive.TUUID ClickHaskell.Statements HS-Source-Dirs: ./ @@ -59,7 +72,7 @@ -- External dependencies , network < 4 - , wide-word < 1 + , wide-word >= 0.1.9 && < 1 ghc-options: -Wno-unticked-promoted-constructors -Wno-dodgy-imports diff --git a/ClickHaskell.hs b/ClickHaskell.hs --- a/ClickHaskell.hs +++ b/ClickHaskell.hs @@ -33,6 +33,11 @@ , passSettings , addSetting + {- ** Parameters -} + , passParameters + , addParameter + , IsParameterType + {- ** SELECT -} {- *** Runner -}, select {- *** Statements -} @@ -48,7 +53,7 @@ , insert , intoTable {- *** Modifiers -} - , ToQueryPart(toQueryPart) + , ToQueryPart(toQueryPart, toQueryPartQuoted) {- ** Ping -}, ping {- ** Commands -}, command, Command @@ -63,8 +68,11 @@ {- * ClickHouse types -} , IsChType(chTypeName, defaultValueOfTypeName) , DateTime, DateTime64 - , Int8, Int16, Int32, Int64, Int128(..) - , UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Word128(..) + , Date + , Int8, Int16, Int32, Int64, Int128(..), Int256(..) + , UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Word128(..), Word256(..) + , Float32, Float64 + , Decimal32, Decimal64, Decimal128, Decimal256 , Nullable , LowCardinality, IsLowCardinalitySupported , UUID @@ -74,11 +82,12 @@ ) where -- Internal -import ClickHaskell.Columns import ClickHaskell.Connection import ClickHaskell.Primitive import ClickHaskell.Statements import ClickHaskell.Protocol +import ClickHaskell.Protocol.Data (UserError (..), Column, SerializableColumn (..), KnownColumn (..)) +import ClickHaskell.Protocol.Client (addParameter, IsParameterType) -- GHC included import Control.Concurrent (newMVar, putMVar, takeMVar) @@ -96,7 +105,7 @@ import System.Environment (lookupEnv) -- External -import Data.WideWord (Int128 (..), Word128 (..)) +import Data.WideWord (Int128 (..), Word128 (..), Int256(..), Word256(..)) -- * Connection @@ -133,19 +142,7 @@ show (DatabaseException err) = "DatabaseException " <> show err <> "\n" <> prettyCallStack callStack show (InternalError err) = "InternalError " <> show err <> "\n" <> prettyCallStack callStack -{- | - Errors intended to be handled by developers --} -data UserError - = UnmatchedType String - -- ^ Column type mismatch in data packet - | UnmatchedColumn String - -- ^ Column name mismatch in data packet - | UnmatchedColumnsCount String - -- ^ Occurs when actual columns count less or more than expected - deriving (Show, Exception) - -- ** Low level -- *** SELECT @@ -163,12 +160,12 @@ ClickHaskell columns output => Select columns output -> Connection -> ([output] -> IO result) -> IO [result] -select (MkSelect mkQuery setts) conn f = do +select (MkSelect mkQuery setts parameters) conn f = do withConnection conn $ \connState -> do writeToConnection connState . flip serialize . mkQueryPacket - . mkQueryArgs connState setts + . mkQueryArgs connState setts parameters . mkQuery $ expectedColumns @columns @output writeToConnection connState (\rev -> serialize rev . Data $ mkDataPacket "" 0 0) @@ -203,12 +200,12 @@ ClickHaskell columns record => Insert columns record -> Connection -> [record] -> IO () -insert (MkInsert mkQuery dbSettings) conn columnsData = do +insert (MkInsert mkQuery dbSettings parameters) conn columnsData = do withConnection conn $ \connState -> do writeToConnection connState . flip serialize . mkQueryPacket - . mkQueryArgs connState dbSettings + . mkQueryArgs connState dbSettings parameters . mkQuery $ expectedColumns @columns @record writeToConnection connState (\rev -> serialize rev . Data $ mkDataPacket "" 0 0) @@ -257,12 +254,12 @@ __Throws exception if any data was returned__ -} command :: HasCallStack => Connection -> Command -> IO () -command conn (MkCommand query settings) = do +command conn (MkCommand query settings parameters) = do withConnection conn $ \connState -> do writeToConnection connState . flip serialize . mkQueryPacket - $ mkQueryArgs connState settings query + $ mkQueryArgs connState settings parameters query writeToConnection connState (\rev -> serialize rev . Data $ mkDataPacket "" 0 0) handleCreate connState where @@ -305,8 +302,8 @@ -- * Internal -mkQueryArgs :: ConnectionState -> DbSettings -> ChString -> QueryPacketArgs -mkQueryArgs MkConnectionState {..} settings query = MkQueryPacketArgs {..} +mkQueryArgs :: ConnectionState -> DbSettings -> QueryParameters -> ChString -> QueryPacketArgs +mkQueryArgs MkConnectionState {..} settings parameters query = MkQueryPacketArgs {..} -- ** Connection @@ -359,7 +356,6 @@ class GClickHaskell (columns :: [Type]) f where - {- Generic deriving can be a bit tricky @@ -385,8 +381,7 @@ GClickHaskell columns (D1 c (C1 c2 f)) where {-# INLINE gDeserializeColumns #-} - gDeserializeColumns doCheck rev size f = - gDeserializeColumns @columns doCheck rev size (f . M1 . M1) + gDeserializeColumns doCheck rev size f = gDeserializeColumns @columns doCheck rev size (f . M1 . M1) {-# INLINE gSerializeRecords #-} gSerializeRecords rev xs f = gSerializeRecords @columns rev xs (unM1 . unM1 . f) @@ -482,30 +477,15 @@ where {-# INLINE gDeserializeColumns #-} gDeserializeColumns doCheck rev size f = do - validateColumnHeader @(Column name chType) doCheck rev =<< deserialize @ColumnHeader rev - deserializeColumn @(Column name chType) rev size (f . M1 . K1 . fromChType) + let errHandler = when doCheck . (throw . UnmatchedResult) + deserializeColumn @(Column name chType) errHandler rev size (f . M1 . K1 . fromChType) {-# INLINE gSerializeRecords #-} - gSerializeRecords rev values f - = serialize rev (mkHeader @(Column name chType)) - <> serializeColumn @(Column name chType) rev (toChType . unK1 . unM1 . f) values + gSerializeRecords rev values f = + serializeColumn @(Column name chType) rev (toChType . unK1 . unM1 . f) values gExpectedColumns = (renderColumnName @(Column name chType), renderColumnType @(Column name chType)) : [] gColumnsCount = 1 - -validateColumnHeader :: forall column . KnownColumn column => Bool -> ProtocolRevision -> ColumnHeader -> Get () -validateColumnHeader doCheck rev MkColumnHeader{..} = do - let expectedColumnName = toChType (renderColumnName @column) - resultColumnName = name - when (doCheck && resultColumnName /= expectedColumnName) $ - throw . UnmatchedResult . UnmatchedColumn - $ "Got column \"" <> show resultColumnName <> "\" but expected \"" <> show expectedColumnName <> "\"" - - let expectedType = fallbackTypeName rev $ toChType (renderColumnType @column) - resultType = fallbackTypeName rev type_ - when (doCheck && resultType /= expectedType) $ - throw . UnmatchedResult . UnmatchedType - $ "Column " <> show resultColumnName <> " has type " <> show resultType <> ". But expected type is " <> show expectedType type family TakeColumn name columns :: Maybe Type diff --git a/ClickHaskell/Columns.hs b/ClickHaskell/Columns.hs deleted file mode 100644 --- a/ClickHaskell/Columns.hs +++ /dev/null @@ -1,288 +0,0 @@ -module ClickHaskell.Columns where - --- Internal -import ClickHaskell.Primitive - --- GHC included -import Data.Binary.Get -import Data.ByteString.Builder -import Data.ByteString.Char8 as BS8 (pack, isPrefixOf) -import Data.Traversable (forM) -import Data.Int -import Data.Kind -import Data.Coerce -import Data.Typeable (Proxy (..)) -import Data.Bits (Bits ((.&.))) -import GHC.Generics (Generic) -import GHC.TypeLits (ErrorMessage (..), KnownSymbol, Symbol, TypeError, symbolVal) - --- External -import Data.WideWord (Int128 (..)) - --- * Column - -data Columns (columns :: [Type]) where - Empty :: Columns '[] - AddColumn - :: KnownColumn (Column name chType) - => Column name chType - -> Columns columns - -> Columns (Column name chType ': columns) - -colLen :: Columns columns -> Int -colLen (AddColumn (col :: col) _) = length $ fromColumn @col col -colLen Empty = 0 - -{- | -Column declaration - -For example: - -@ -type MyColumn = Column "myColumn" ChString -@ --} -data Column (name :: Symbol) (chType :: Type) where - UInt8Column :: [UInt8] -> Column name UInt8 - Int8Column :: [Int8] -> Column name Int8 - UInt16Column :: [UInt16] -> Column name UInt16 - Int16Column :: [Int16] -> Column name Int16 - UInt32Column :: [UInt32] -> Column name UInt32 - Int32Column :: [Int32] -> Column name Int32 - UInt64Column :: [UInt64] -> Column name UInt64 - Int64Column :: [Int64] -> Column name Int64 - UInt128Column :: [UInt128] -> Column name UInt128 - Int128Column :: [Int128] -> Column name Int128 - UInt256Column :: [UInt256] -> Column name UInt256 - DateTimeColumn :: [DateTime tz] -> Column name (DateTime tz) - DateTime64Column :: [DateTime64 precision tz] -> Column name (DateTime64 precision tz) - BoolColumn :: [Bool] -> Column name Bool - Enum8Column :: [Enum8 enums] -> Column name (Enum8 enums) - Enum16Column :: [Enum16 enums] -> Column name (Enum16 enums) - DateColumn :: [Date] -> Column name Date - UUIDColumn :: [UUID] -> Column name UUID - StringColumn :: [ChString] -> Column name ChString - ArrayColumn :: [Array chType] -> Column name (Array chType) - NullableColumn :: [Nullable chType] -> Column name (Nullable chType) - LowCardinalityColumn :: [chType] -> Column name (LowCardinality chType) - -type family GetColumnName column :: Symbol where GetColumnName (Column name columnType) = name -type family GetColumnType column :: Type where GetColumnType (Column name columnType) = columnType - -data ColumnHeader = MkColumnHeader - { name :: ChString - , type_ :: ChString - , is_custom :: UInt8 `SinceRevision` DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION - } deriving (Generic, Serializable) - -fallbackTypeName :: ProtocolRevision -> ChString -> ChString -fallbackTypeName rev (MkChString typeName) = MkChString $ - if rev < mkRev @DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE - && isPrefixOf "DateTime(" typeName - then "DateTime" - else typeName - -class - ( IsChType (GetColumnType column) - , KnownSymbol (GetColumnName column) - ) => - KnownColumn column where - renderColumnName :: Builder - renderColumnName = (stringUtf8 . symbolVal @(GetColumnName column)) Proxy - - renderColumnType :: Builder - renderColumnType = byteString . BS8.pack $ chTypeName @(GetColumnType column) - - mkHeader :: ColumnHeader - mkHeader = let - name = toChType $ renderColumnName @column - type_ = toChType $ chTypeName @(GetColumnType column) - is_custom = AfterRevision 0 - in MkColumnHeader{..} - - toColumn :: [GetColumnType column] -> Column (GetColumnName column) (GetColumnType column) - fromColumn :: Column (GetColumnName column) (GetColumnType column) -> [GetColumnType column] - -instance KnownSymbol name => KnownColumn (Column name UInt8) where - toColumn = UInt8Column - fromColumn (UInt8Column values) = values - -instance KnownSymbol name => KnownColumn (Column name UInt16) where - toColumn = UInt16Column - fromColumn (UInt16Column values) = values - -instance KnownSymbol name => KnownColumn (Column name UInt32) where - toColumn = UInt32Column - fromColumn (UInt32Column values) = values - -instance KnownSymbol name => KnownColumn (Column name UInt64) where - toColumn = UInt64Column - fromColumn (UInt64Column values) = values - -instance KnownSymbol name => KnownColumn (Column name UInt128) where - toColumn = UInt128Column - fromColumn (UInt128Column values) = values - -instance KnownSymbol name => KnownColumn (Column name UInt256) where - toColumn = UInt256Column - fromColumn (UInt256Column values) = values - -instance KnownSymbol name => KnownColumn (Column name Int8) where - toColumn = Int8Column - fromColumn (Int8Column values) = values - -instance KnownSymbol name => KnownColumn (Column name Int16) where - toColumn = Int16Column - fromColumn (Int16Column values) = values - -instance KnownSymbol name => KnownColumn (Column name Int32) where - toColumn = Int32Column - fromColumn (Int32Column values) = values - -instance KnownSymbol name => KnownColumn (Column name Int64) where - toColumn = Int64Column - fromColumn (Int64Column values) = values - -instance KnownSymbol name => KnownColumn (Column name Int128) where - toColumn = Int128Column - fromColumn (Int128Column values) = values - -instance KnownSymbol name => KnownColumn (Column name Date) where - toColumn = DateColumn - fromColumn (DateColumn values) = values - -instance - ( KnownSymbol name - , IsChType (DateTime tz) - ) => - KnownColumn (Column name (DateTime tz)) - where - toColumn = DateTimeColumn - fromColumn (DateTimeColumn values) = values - -instance - ( KnownSymbol name - , IsChType (DateTime64 precision tz) - ) => - KnownColumn (Column name (DateTime64 precision tz)) - where - toColumn = DateTime64Column - fromColumn (DateTime64Column values) = values - -instance - ( KnownSymbol name - , IsChType Bool - ) => - KnownColumn (Column name Bool) - where - toColumn = BoolColumn - fromColumn (BoolColumn values) = values - -instance (KnownSymbol name, KnownSymbol enums) => KnownColumn (Column name (Enum8 enums)) where - toColumn = Enum8Column - fromColumn (Enum8Column values) = values - -instance (KnownSymbol name, KnownSymbol enums) => KnownColumn (Column name (Enum16 enums)) where - toColumn = Enum16Column - fromColumn (Enum16Column values) = values - -instance KnownSymbol name => KnownColumn (Column name UUID) where - toColumn = UUIDColumn - fromColumn (UUIDColumn values) = values - -instance - ( KnownSymbol name - , IsChType chType - , IsChType (Nullable chType) - ) => - KnownColumn (Column name (Nullable chType)) - where - toColumn = NullableColumn - fromColumn (NullableColumn values) = values - -instance KnownSymbol name => KnownColumn (Column name ChString) where - toColumn = StringColumn - fromColumn (StringColumn values) = values - -instance - ( KnownSymbol name - , IsChType (LowCardinality chType) - , IsLowCardinalitySupported chType - ) => - KnownColumn (Column name (LowCardinality chType)) - where - toColumn = LowCardinalityColumn . map coerce - fromColumn (LowCardinalityColumn values) = map coerce values - -instance (KnownSymbol name, IsChType chType) => KnownColumn (Column name (Array chType)) where - toColumn = ArrayColumn - fromColumn (ArrayColumn values) = values - - -class KnownColumn column => SerializableColumn column where - deserializeColumn :: ProtocolRevision -> UVarInt -> (GetColumnType column -> a) -> Get [a] - serializeColumn :: ProtocolRevision -> (a -> GetColumnType column) -> [a] -> Builder - -instance - ( KnownColumn (Column name chType) - , Serializable chType - , IsChType chType - ) => - SerializableColumn (Column name chType) where - {-# INLINE deserializeColumn #-} - deserializeColumn rev rows f = map f <$> replicateGet rev rows - - {-# INLINE serializeColumn #-} - serializeColumn rev f column = foldMap (serialize @chType rev . f) column - -instance {-# OVERLAPPING #-} - ( KnownColumn (Column name (Nullable chType)) - , Serializable chType - , IsChType chType - ) => - SerializableColumn (Column name (Nullable chType)) where - {-# INLINE deserializeColumn #-} - deserializeColumn rev rows f = do - nulls <- replicateGet @UInt8 rev rows - forM nulls (\nulFlag -> case nulFlag of - 0 -> f . Just <$> deserialize @chType rev - _ -> (f Nothing <$ deserialize @chType rev) - ) - - {-# INLINE serializeColumn #-} - serializeColumn rev f column - = foldMap (serialize @UInt8 rev . maybe 1 (const 0) . f) column - <> foldMap (serialize @chType rev . maybe defaultValueOfTypeName id . f) column - -instance {-# OVERLAPPING #-} - ( KnownColumn (Column name (LowCardinality chType)) - , Serializable chType - , IsLowCardinalitySupported chType - , TypeError ('Text "LowCardinality deserialization still unsupported") - ) => - SerializableColumn (Column name (LowCardinality chType)) where - {-# INLINE deserializeColumn #-} - deserializeColumn rev rows f = do - _serializationType <- (.&. 0xf) <$> deserialize @UInt64 rev - _index_size <- deserialize @Int64 rev - -- error $ "Trace | " <> show _serializationType <> " : " <> show _index_size - map f . coerce - <$> replicateGet @chType rev rows - - {-# INLINE serializeColumn #-} - serializeColumn _rev column = undefined column - -instance {-# OVERLAPPING #-} - ( KnownColumn (Column name (Array chType)) - , Serializable chType - ) - => SerializableColumn (Column name (Array chType)) where - {-# INLINE deserializeColumn #-} - deserializeColumn rev rows f = do - offsets <- replicateGet @UInt64 rev rows - forM offsets (fmap (f . MkChArray) . replicateGet @chType rev . fromIntegral) - - {-# INLINE serializeColumn #-} - serializeColumn rev f column - = foldMap (serialize @UInt64 rev . fromIntegral . length . f) column - <> foldMap (foldMap (serialize @chType rev) . f) column diff --git a/ClickHaskell/Connection.hs b/ClickHaskell/Connection.hs --- a/ClickHaskell/Connection.hs +++ b/ClickHaskell/Connection.hs @@ -8,7 +8,7 @@ import Control.Exception (SomeException, bracketOnError, catch, finally, throwIO) import Data.Binary.Builder (Builder, toLazyByteString) import Data.ByteString as BS (ByteString, null) -import Data.IORef (atomicWriteIORef, newIORef, readIORef) +import Data.IORef (newIORef, readIORef, writeIORef) import Data.Maybe (fromMaybe) import GHC.Exception (Exception) @@ -84,8 +84,8 @@ mkBuffer :: BufferArgs -> IO Buffer mkBuffer MkBufferArgs{..} = do - buff <- newIORef "" - let writeBuff bs = atomicWriteIORef buff bs + buff <- newIORef mempty + let writeBuff bs = writeIORef buff bs pure MkBuffer { writeConn = writeSock @@ -93,7 +93,7 @@ , readBuff = do currentBuffer <- readIORef buff if (not . BS.null) currentBuffer - then writeBuff "" *> pure currentBuffer + then writeBuff mempty *> pure currentBuffer else do sockBytes <- readSock if BS.null sockBytes @@ -101,7 +101,7 @@ else pure sockBytes , destroyBuff = do closeSock - writeBuff "" + writeBuff mempty } data BufferArgs = MkBufferArgs diff --git a/ClickHaskell/Primitive.hs b/ClickHaskell/Primitive.hs --- a/ClickHaskell/Primitive.hs +++ b/ClickHaskell/Primitive.hs @@ -1,815 +1,33 @@ {-# LANGUAGE BangPatterns #-} -module ClickHaskell.Primitive where - --- GHC included -import Control.Applicative (liftA2) -import Control.DeepSeq (NFData) -import Data.Binary.Get -import Data.Bits (Bits (setBit, unsafeShiftL, unsafeShiftR, (.&.), (.|.))) -import Data.Bool (bool) -import Data.ByteString as BS (ByteString, length) -import Data.ByteString.Builder -import Data.ByteString.Char8 as BS8 (pack, unpack, concatMap, singleton, replicate, length) -import Data.ByteString.Lazy (toStrict) -import Data.Coerce (coerce) -import Data.Int (Int16, Int32, Int64, Int8) -import Data.List (uncons) -import Data.String (IsString (..)) -import Data.Time (UTCTime) -import Data.Time.Clock.POSIX -import Data.Type.Bool (Not) -import Data.Type.Equality (type (==)) -import Data.Typeable (Proxy (..)) -import Data.Word (Word16, Word32, Word64, Word8) -import GHC.Generics (C1, D1, Generic (..), K1 (K1), M1 (M1), Meta (MetaSel), Rec0, S1, type (:*:) (..)) -import GHC.TypeLits (ErrorMessage (..), KnownNat, KnownSymbol, Nat, Symbol, TypeError, natVal, symbolVal) -import Prelude hiding (liftA2) - --- External -import Data.WideWord (Int128 (..), Word256(..), Word128(..)) - --- * User types - --- ** Abstractions - -class Serializable chType - where - default serialize :: (Generic chType, GSerial (Rep chType)) => ProtocolRevision -> chType -> Builder - serialize :: ProtocolRevision -> chType -> Builder - serialize rev = gSerialize rev . from - - {-# INLINE deserialize #-} - default deserialize :: (Generic chType, GSerial (Rep chType)) => ProtocolRevision -> Get chType - deserialize :: ProtocolRevision -> Get chType - deserialize rev = to <$> gDeserialize rev - -{-# INLINE replicateGet #-} -replicateGet :: Serializable chType => ProtocolRevision -> UVarInt -> Get [chType] -replicateGet rev cnt0 = loopGet cnt0 - where - loopGet cnt - | cnt == 0 = pure [] - | otherwise = liftA2 (:) (deserialize rev) (loopGet (cnt - 1)) - -instance Serializable prim => Serializable [prim] where - serialize rev list - = serialize @UVarInt rev (fromIntegral $ Prelude.length list) - <> foldMap (serialize @prim rev) list - deserialize rev = do - len <- deserialize @UVarInt rev - replicateGet @prim rev len - {-# INLINE deserialize #-} - -class ToQueryPart chType where - toQueryPart :: chType -> Builder - -class IsChType chType - where - -- | Shows database original type name - -- - -- @ - -- chTypeName \@ChString = \"String\" - -- chTypeName \@(Nullable UInt32) = \"Nullable(UInt32)\" - -- @ - chTypeName :: String - - defaultValueOfTypeName :: chType - -class ToChType chType userType where - toChType :: userType -> chType - fromChType :: chType -> userType - -instance {-# OVERLAPPABLE #-} (IsChType chType, chType ~ inputType) => ToChType chType inputType where - toChType = id - fromChType = id - - --- ** Int8 - -instance IsChType Int8 where - chTypeName = "Int8" - defaultValueOfTypeName = 0 - -instance Serializable Int8 where - serialize _ = int8 - deserialize _ = getInt8 - {-# INLINE deserialize #-} - -instance ToQueryPart Int8 where - toQueryPart = byteString . BS8.pack . show - - --- ** Int16 - -instance IsChType Int16 where - chTypeName = "Int16" - defaultValueOfTypeName = 0 - -instance Serializable Int16 where - serialize _ = int16LE - deserialize _ = getInt16le - {-# INLINE deserialize #-} - -instance ToQueryPart Int16 where - toQueryPart = byteString . BS8.pack . show - - --- ** Int32 - -instance IsChType Int32 where - chTypeName = "Int32" - defaultValueOfTypeName = 0 - -instance Serializable Int32 where - serialize _ = int32LE - deserialize _ = getInt32le - {-# INLINE deserialize #-} - -instance ToQueryPart Int32 where - toQueryPart = byteString . BS8.pack . show - - --- ** Int64 - -instance IsChType Int64 where - chTypeName = "Int64" - defaultValueOfTypeName = 0 - -instance Serializable Int64 where - serialize _ = int64LE - deserialize _ = getInt64le - {-# INLINE deserialize #-} - -instance ToQueryPart Int64 where - toQueryPart = byteString . BS8.pack . show - - --- ** Int128 - -instance IsChType Int128 where - chTypeName = "Int128" - defaultValueOfTypeName = 0 - -instance Serializable Int128 where - serialize _ = (\(Int128 hi lo) -> word64LE lo <> word64LE hi) - deserialize _ = do - low <- getWord64le - high <- getWord64le - pure $ Int128 high low - {-# INLINE deserialize #-} - -instance ToQueryPart Int128 where - toQueryPart = byteString . BS8.pack . show - - --- ** UInt8 - -{- | ClickHouse UInt8 column type -} -type UInt8 = Word8 -instance IsChType UInt8 where - chTypeName = "UInt8" - defaultValueOfTypeName = 0 - -instance Serializable UInt8 where - serialize _ = word8 - deserialize _ = getWord8 - {-# INLINE deserialize #-} - -instance ToQueryPart UInt8 where - toQueryPart = byteString . BS8.pack . show - - --- ** UInt16 - -{- | ClickHouse UInt16 column type -} -type UInt16 = Word16 -instance IsChType UInt16 where - chTypeName = "UInt16" - defaultValueOfTypeName = 0 - -instance Serializable UInt16 where - serialize _ = word16LE - deserialize _ = getWord16le - {-# INLINE deserialize #-} - -instance ToQueryPart UInt16 where - toQueryPart = byteString . BS8.pack . show - - --- ** UInt32 - -{- | ClickHouse UInt32 column type -} -type UInt32 = Word32 -instance IsChType UInt32 where - chTypeName = "UInt32" - defaultValueOfTypeName = 0 - -instance Serializable UInt32 where - serialize _ = word32LE - deserialize _ = getWord32le - {-# INLINE deserialize #-} - -instance ToQueryPart UInt32 where - toQueryPart = byteString . BS8.pack . show - - --- ** UInt64 - -{- | ClickHouse UInt64 column type -} -type UInt64 = Word64 -instance IsChType UInt64 where - chTypeName = "UInt64" - defaultValueOfTypeName = 0 - -instance Serializable UInt64 where - serialize _ = word64LE - deserialize _ = getWord64le - {-# INLINE deserialize #-} - -instance ToQueryPart UInt64 where - toQueryPart = byteString . BS8.pack . show - - --- ** UInt128 - -{- | ClickHouse UInt128 column type -} -type UInt128 = Word128 -instance IsChType UInt128 where - chTypeName = "UInt128" - defaultValueOfTypeName = 0 - -instance Serializable UInt128 where - serialize _ = (\(Word128 hi lo) -> word64LE lo <> word64LE hi) - deserialize _ = do - low <- getWord64le - high <- getWord64le - pure $ Word128 high low - {-# INLINE deserialize #-} - -instance ToQueryPart UInt128 where - toQueryPart w128 = "'" <> (byteString . BS8.pack . show) w128 <> "'" - - --- ** Int256 - -{- | ClickHouse UInt128 column type -} -type UInt256 = Word256 -instance IsChType UInt256 where - chTypeName = "UInt256" - defaultValueOfTypeName = 0 - -instance Serializable UInt256 where - serialize _ = (\(Word256 high mid1 mid0 low) -> word64LE low <> word64LE mid0 <> word64LE mid1 <> word64LE high) - deserialize _ = do - low <- getWord64le - mid0 <- getWord64le - mid1 <- getWord64le - high <- getWord64le - pure $ Word256 high mid1 mid0 low - {-# INLINE deserialize #-} - -instance ToQueryPart UInt256 where - toQueryPart w256 = "'" <> (byteString . BS8.pack . show) w256 <> "'" - - --- ** Date - -{- | ClickHouse Date column type -} -newtype Date = MkDate Word16 - deriving newtype (Show, Eq, Bits, Bounded, Enum, NFData, Num) - -instance IsChType Date where - chTypeName = "Date" - defaultValueOfTypeName = 0 - -instance Serializable Date where - serialize _ (MkDate w16) = word16LE w16 - deserialize _ = MkDate <$> getWord16le - {-# INLINE deserialize #-} - -instance ToChType Date Word16 where - toChType = MkDate - fromChType (MkDate w16) = w16 - - --- ** ChString - -{- | ClickHouse String column type -} -newtype ChString = MkChString BS.ByteString - deriving newtype (Show, Eq, IsString, NFData) - -instance IsChType ChString where - chTypeName = "String" - defaultValueOfTypeName = "" - -instance Serializable ChString where - serialize rev (MkChString str) = (serialize @UVarInt rev . fromIntegral . BS.length) str <> byteString str - deserialize rev = do - len <- deserialize @UVarInt rev - MkChString <$> (getByteString . fromIntegral) len - {-# INLINE deserialize #-} - -instance ToChType ChString BS.ByteString where - toChType = MkChString - fromChType (MkChString string) = string - -instance ToChType ChString Builder where - toChType = MkChString . toStrict . toLazyByteString - fromChType (MkChString string) = byteString string - -instance ToChType ChString String where - toChType = MkChString . BS8.pack - fromChType (MkChString bs)= BS8.unpack bs - -instance ToQueryPart ChString where - toQueryPart (MkChString string) = "'" <> escapeQuery string <> "'" - where - escapeQuery :: BS.ByteString -> Builder - escapeQuery = byteString . BS8.concatMap (\sym -> - case sym of - '\'' -> "\\\'" - '\\' -> "\\\\" - _ -> BS8.singleton sym - ) - - - --- ** UUID - -{- | ClickHouse UUID column type -} -newtype UUID = MkUUID Word128 - deriving newtype (Generic, Show, Eq, NFData, Bounded, Enum, Num) -instance IsChType UUID where - chTypeName = "UUID" - defaultValueOfTypeName = 0 - -instance Serializable UUID where - serialize _ = (\(MkUUID (Word128 hi lo)) -> word64LE lo <> word64LE hi) - deserialize _ = do - low <- getWord64le - high <- getWord64le - pure $ MkUUID (Word128 high low) - {-# INLINE deserialize #-} - -instance ToChType UUID (Word64, Word64) where - toChType = MkUUID . uncurry (flip Word128) - fromChType (MkUUID (Word128 w64hi w64lo)) = (w64hi, w64lo) - -instance ToQueryPart UUID where - toQueryPart (MkUUID (Word128 hi lo)) = mconcat - ["'", p 3 hi, p 2 hi, "-", p 1 hi, "-", p 0 hi, "-", p 3 lo, "-", p 2 lo, p 1 lo, p 0 lo, "'"] - where - p :: Int -> Word64 -> Builder - p shiftN word = word16HexFixed $ fromIntegral (word `unsafeShiftR` (shiftN*16)) - - --- ** Nullable - -{- | ClickHouse Nullable(T) column type - (type synonym for Maybe) - -} -type Nullable = Maybe -instance IsChType chType => IsChType (Nullable chType) - where - chTypeName = "Nullable(" <> chTypeName @chType <> ")" - defaultValueOfTypeName = Nothing - -instance - ToChType inputType chType - => - ToChType (Nullable inputType) (Nullable chType) - where - toChType = fmap (toChType @inputType @chType) - fromChType = fmap (fromChType @inputType) - -instance ToQueryPart chType => ToQueryPart (Nullable chType) - where - toQueryPart = maybe "null" toQueryPart - - --- ** Enum8 - -newtype Enum8 (enums :: Symbol) = MkEnum8 Int8 - deriving newtype (Serializable, Show, Eq, Num, Bits, Bounded, Enum) - -instance KnownSymbol enums => IsChType (Enum8 enums) where - chTypeName = "Enum8(" <> symbolVal @enums Proxy <> ")" - defaultValueOfTypeName = 0 - -instance ToChType (Enum8 enums) Int8 where - toChType = MkEnum8 - fromChType (MkEnum8 i8)= i8 - -instance ToQueryPart (Enum8 enums) where - toQueryPart = toQueryPart . fromChType @_ @Int8 - - --- ** Enum16 - -newtype Enum16 (enums :: Symbol) = MkEnum16 Int16 - deriving newtype (Serializable, Show, Eq, Num, Bits, Bounded, Enum) - -instance KnownSymbol enums => IsChType (Enum16 enums) where - chTypeName = "Enum16(" <> symbolVal @enums Proxy <> ")" - defaultValueOfTypeName = 0 - -instance ToChType (Enum16 enums) Int16 where - toChType = MkEnum16 - fromChType (MkEnum16 i16) = i16 - -instance ToQueryPart (Enum16 enums) where - toQueryPart = toQueryPart . fromChType @_ @Int16 - - --- ** Bool - -instance IsChType Bool where - chTypeName = "Bool" - defaultValueOfTypeName = False - -instance Serializable Bool where - serialize _ = int8 . bool 0 1 - deserialize _ = (\int -> case int of 0->False; _->True) <$> getInt8 - {-# INLINE deserialize #-} - -instance ToQueryPart Bool where - toQueryPart = bool "false" "true" - - --- ** DateTime - -{- | -ClickHouse DateTime column type (parametrized with timezone) - ->>> chTypeName @(DateTime "") -"DateTime" ->>> chTypeName @(DateTime "UTC") -"DateTime('UTC')" - -__Note:__ 'DateTime' stores whole seconds only, so converting from 'UTCTime' \ -will drop any sub-second precision. - ->>> let myUtcTime = posixSecondsToUTCTime 0.042_042 ->>> toChType @(DateTime "") @UTCTime myUtcTime -0 --} -newtype DateTime (tz :: Symbol) = MkDateTime Word32 - deriving newtype (Show, Eq, Num, Bits, Enum, Ord, Real, Integral, Bounded, NFData) - -instance KnownSymbol tz => IsChType (DateTime tz) - where - chTypeName = case symbolVal @tz Proxy of - "" -> "DateTime" - tz -> "DateTime('" <> tz <> "')" - defaultValueOfTypeName = MkDateTime 0 - -instance Serializable (DateTime tz) where - serialize _ (MkDateTime w32) = word32LE w32 - deserialize _ = MkDateTime <$> getWord32le - {-# INLINE deserialize #-} - -instance ToChType (DateTime tz) Word32 where - toChType = MkDateTime - fromChType (MkDateTime w32)= w32 - -instance ToChType (DateTime tz) UTCTime where - toChType = MkDateTime . floor . utcTimeToPOSIXSeconds - fromChType (MkDateTime w32) = posixSecondsToUTCTime (fromIntegral w32) - -instance ToQueryPart (DateTime tz) - where - toQueryPart chDateTime = let time = BS8.pack . show . coerce @(DateTime tz) @Word32 $ chDateTime - in byteString (BS8.replicate (10 - BS8.length time) '0' <> time) - - --- ** DateTime64 - -{- | -ClickHouse DateTime64 column type (parametrized with timezone) - ->>> chTypeName @(DateTime64 3 "") -"DateTime64(3)" ->>> chTypeName @(DateTime64 3 "UTC") -"DateTime64(3, 'UTC')" - -__Note:__ conversion from 'UTCTime' may lose sub-second precision if \ -the @precision@ parameter is lower than the actual timestamp precision. - ->>> let myUtcTime = posixSecondsToUTCTime 42.000_000_042 ->>> toChType @(DateTime64 6 "") @UTCTime myUtcTime -42000000 ->>> toChType @(DateTime64 9 "") @UTCTime myUtcTime -42000000042 --} -newtype DateTime64 (precision :: Nat) (tz :: Symbol) = MkDateTime64 Word64 - deriving newtype (Show, Eq, Num, Bits, Enum, Ord, Real, Integral, Bounded, NFData) - -instance - (KnownSymbol tz, KnownNat precision) - => - IsChType (DateTime64 precision tz) - where - chTypeName = - let - prec = show (natVal @precision Proxy) - in - case symbolVal @tz Proxy of - "" -> "DateTime64(" <> prec <> ")" - tz -> "DateTime64(" <> prec <> ", '" <> tz <> "')" - defaultValueOfTypeName = MkDateTime64 0 - -instance Serializable (DateTime64 precision tz) where - serialize _ (MkDateTime64 w64) = word64LE w64 - deserialize _ = MkDateTime64 <$> getWord64le - {-# INLINE deserialize #-} - -instance ToChType (DateTime64 precision tz) Word64 where - toChType = MkDateTime64 - fromChType (MkDateTime64 w64) = w64 - -instance KnownNat precision => ToChType (DateTime64 precision tz) UTCTime where - toChType = MkDateTime64 . floor . (* (10 ^ natVal (Proxy @precision))) - . utcTimeToPOSIXSeconds - fromChType (MkDateTime64 w64) = posixSecondsToUTCTime - $ (/ (10 ^ natVal (Proxy @precision))) $ fromIntegral w64 - --- ToDo: Need to be fixed --- instance ToQueryPart (DateTime64 precision tz) --- where --- toQueryPart chDateTime = --- let time = BS8.pack . show . fromChType @_ @Word64 $ chDateTime --- in byteString (BS8.replicate (12 - BS8.length time) '0' <> time) - - --- ** Array - --- | ClickHouse Array column type -newtype Array a = MkChArray [a] - deriving newtype (Show, Eq, NFData, Foldable) -instance IsChType chType => IsChType (Array chType) - where - chTypeName = "Array(" <> chTypeName @chType <> ")" - defaultValueOfTypeName = MkChArray [] - -instance ToChType chType inputType => ToChType (Array chType) [inputType] - where - toChType = MkChArray . map toChType - fromChType (MkChArray values) = map fromChType values - -instance (IsChType chType, ToQueryPart chType) => ToQueryPart (Array chType) - where - toQueryPart - = (\x -> "[" <> x <> "]") - . (maybe "" (uncurry (foldl (\a b -> a <> "," <> b))) . uncons - . map (toQueryPart @chType)) . coerce @(Array chType) @[chType] - - --- ** LowCardinality - --- | ClickHouse LowCardinality(T) column type -newtype LowCardinality chType = MkLowCardinality chType -instance IsLowCardinalitySupported chType => IsChType (LowCardinality chType) - where - chTypeName = "LowCardinality(" <> chTypeName @chType <> ")" - defaultValueOfTypeName = MkLowCardinality $ defaultValueOfTypeName @chType - -deriving newtype instance (Eq chType, IsLowCardinalitySupported chType) => Eq (LowCardinality chType) -deriving newtype instance (NFData chType, IsLowCardinalitySupported chType) => NFData (LowCardinality chType) -deriving newtype instance IsString (LowCardinality ChString) - -class IsChType chType => IsLowCardinalitySupported chType -instance IsLowCardinalitySupported ChString -instance - ( IsLowCardinalitySupported chType - , IsChType (Nullable chType) - ) => - IsLowCardinalitySupported (Nullable chType) - -instance {-# OVERLAPPABLE #-} - ( IsChType chType - , TypeError - ( 'Text "LowCardinality(" ':<>: 'ShowType chType ':<>: 'Text ") is unsupported" - ':$$: 'Text "Use one of these types:" - ':$$: 'Text " ChString" - ':$$: 'Text " DateTime" - ':$$: 'Text " Nullable(T)" - ) - ) => IsLowCardinalitySupported chType - -instance - ToChType inputType chType - => - ToChType (LowCardinality inputType) chType where - toChType = MkLowCardinality . toChType - fromChType (MkLowCardinality lc)= fromChType @inputType lc - -instance ToQueryPart chType => ToQueryPart (LowCardinality chType) - where - toQueryPart (MkLowCardinality chType) = toQueryPart chType - - --- ** Generics - -class GSerial f where - gSerialize :: ProtocolRevision -> f p -> Builder - gDeserialize :: ProtocolRevision -> Get (f p) - -instance GSerial f => GSerial (D1 c (C1 c2 f)) where - gSerialize rev (M1 (M1 re)) = gSerialize rev re - {-# INLINE gSerialize #-} - gDeserialize rev = M1 . M1 <$> gDeserialize rev - {-# INLINE gDeserialize #-} - -instance (GSerial left1, GSerial right) => GSerial (left1 :*: right) where - gSerialize rev (l :*: r) = gSerialize rev l <> gSerialize rev r - {-# INLINE gSerialize #-} - gDeserialize rev = do - liftA2 (:*:) - (gDeserialize rev) - (gDeserialize rev) - {-# INLINE gDeserialize #-} - -instance - (Serializable chType, Not (sel == "server_revision") ~ True) - => - GSerial (S1 ('MetaSel ('Just sel) a b c) (Rec0 chType)) where - gSerialize rev (M1 (K1 re)) = serialize rev re - {-# INLINE gSerialize #-} - gDeserialize rev = M1 . K1 <$> deserialize @chType rev - {-# INLINE gDeserialize #-} - - -instance {-# OVERLAPPING #-} - GSerial right - => - GSerial (S1 ('MetaSel ('Just "server_revision") a b c) (Rec0 ProtocolRevision) :*: right) - where - gSerialize rev (M1 (K1 (MkProtocolRevision server_rev)) :*: right)= do - serialize rev server_rev <> gSerialize rev right - {-# INLINE gSerialize #-} - gDeserialize rev = do - chosenRev <- min rev . MkProtocolRevision <$> deserialize @UVarInt rev - liftA2 (:*:) - (pure . M1 . K1 $ chosenRev) - (gDeserialize @right chosenRev) - {-# INLINE gDeserialize #-} - - - - --- * Protocol parts - --- ** UVarInt - -{- | - Unsigned variable-length quantity encoding - - Part of protocol implementation --} -newtype UVarInt = MkUVarInt Word64 - deriving newtype (Show, Eq, Num, Bits, Enum, Ord, Real, Integral, Bounded, NFData) - -instance Serializable UVarInt where - serialize _ = goUVarIntSer - where - goUVarIntSer i - | i < 0x80 = word8 (fromIntegral i) - | otherwise = word8 (setBit (fromIntegral i) 7) <> goUVarIntSer (i `unsafeShiftR` 7) - deserialize _ = goUVarIntDeser 0 (0 :: UVarInt) - where - goUVarIntDeser i o | i < 10 = do - byte <- getWord8 - let o' = o .|. ((fromIntegral byte .&. 0x7f) `unsafeShiftL` (7 * i)) - if byte .&. 0x80 == 0 then pure $! o' else goUVarIntDeser (i + 1) $! o' - goUVarIntDeser _ _ = fail "input exceeds varuint size" - {-# INLINE deserialize #-} - - --- ** Versioning - --- >>> show major <> "." <> show minor <> "." <> show patch --- "1.0.0" -major, minor, patch :: UVarInt -major = 1 -minor = 0 -patch = 0 - -clientName :: ChString -clientName = "ClickHaskell" - -newtype ProtocolRevision = MkProtocolRevision UVarInt - deriving newtype (Eq, Num, Ord, Serializable) - -mkRev :: forall nat . KnownNat nat => ProtocolRevision -mkRev = (fromIntegral . natVal) (Proxy @nat) - -{- | -Protocol implementation part for backward compatilibity formalization - -NB: - - Be carefull with `BeforeRevision` value. - - If **chosen protocol** revision would be >= **revisionNumber** - then you would get an exception during serialization. - - To avoid this: - - 1. On client side - provide `AfterRevision` with some empty value - 2. On proxy side - provide server-to-server packets mapping with fallbacks on revision upgrade - --} -data Revisioned (revisionNumber :: Nat) b a = BeforeRevision b | AfterRevision a - -type SinceRevision after rev = Revisioned rev () after - -instance - (KnownNat revision, Serializable before, Serializable after) - => - Serializable (Revisioned revision before after) - where - serialize rev sinceRevVal = - if rev < mkRev @revision - then mempty - else case sinceRevVal of - BeforeRevision _b -> error "Protocol-specific implementation error" -- Watch `Revisioned` note - AfterRevision a -> serialize rev a - deserialize rev = - if rev < mkRev @revision - then BeforeRevision <$> deserialize @before rev - else AfterRevision <$> deserialize @after rev - -instance Serializable () where - serialize _ () = "" - deserialize _ = pure () +module ClickHaskell.Primitive + ( module ClickHaskell.Primitive.Serialization + , module ClickHaskell.Primitive.TArray + , module ClickHaskell.Primitive.TDate + , module ClickHaskell.Primitive.TDateTime + , module ClickHaskell.Primitive.TDecimal + , module ClickHaskell.Primitive.TEnum + , module ClickHaskell.Primitive.TFloat + , module ClickHaskell.Primitive.TLowCardinality + , module ClickHaskell.Primitive.TString + , module ClickHaskell.Primitive.TNullable + , module ClickHaskell.Primitive.TUInt + , module ClickHaskell.Primitive.TUUID -{- - Slightly modified C++ sources: - https://github.com/ClickHouse/ClickHouse/blob/eb4a74d7412a1fcf52727cd8b00b365d6b9ed86c/src/Core/ProtocolDefines.h#L6 --} -type DBMS_TCP_PROTOCOL_VERSION = DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS; + ) where -type DBMS_MIN_REVISION_WITH_CLIENT_INFO = 54032; -type DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE = 54058; -type DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO = 54060; -type DBMS_MIN_REVISION_WITH_TABLES_STATUS = 54226; -type DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE = 54337; -type DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME = 54372; -type DBMS_MIN_REVISION_WITH_VERSION_PATCH = 54401; -type DBMS_MIN_REVISION_WITH_SERVER_LOGS = 54406; -type DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD = 54448; -type DBMS_MIN_MAJOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD = 21; -type DBMS_MIN_MINOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD = 4; -type DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA = 54410; -type DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE = 54405; -type DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO = 54420; -type DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS = 54429; -type DBMS_MIN_REVISION_WITH_SCALARS = 54429; -type DBMS_MIN_REVISION_WITH_OPENTELEMETRY = 54442; -type DBMS_MIN_REVISION_WITH_AGGREGATE_FUNCTIONS_VERSIONING = 54452; -type DBMS_CLUSTER_INITIAL_PROCESSING_PROTOCOL_VERSION = 1; -type DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA = 2; -type DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = 2; -type DATA_LAKE_TABLE_STATE_SNAPSHOT_PROTOCOL_VERSION = 1; -type DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3; -type DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_MARK_SEGMENT_SIZE_FIELD = 4; -type DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_PROJECTION = 5; -type DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 5; -type DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453; -type DBMS_MIN_REVISION_WITH_QUERY_AND_LINE_NUMBERS = 54475; -type DBMS_MERGE_TREE_PART_INFO_VERSION = 1; -type DBMS_QUERY_PLAN_SERIALIZATION_VERSION = 0; -type DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET = 54441; -type DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO = 54443; -type DBMS_MIN_REVISION_WITH_REFERER_IN_CLIENT_INFO = 54447; -type DBMS_MIN_PROTOCOL_VERSION_WITH_DISTRIBUTED_DEPTH = 54448; -type DBMS_MIN_PROTOCOL_VERSION_WITH_INCREMENTAL_PROFILE_EVENTS = 54451; -type DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION = 54454; -type DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME = 54449; -type DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT = 54456; -type DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED = 54457; -type DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM = 54458; -type DBMS_MIN_PROTOCOL_VERSION_WITH_QUOTA_KEY = 54458; -type DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS = 54459; -type DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS = 54460; -type DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES = 54461; -type DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 = 54462; -type DBMS_MIN_PROTOCOL_VERSION_WITH_TOTAL_BYTES_IN_PROGRESS = 54463; -type DBMS_MIN_PROTOCOL_VERSION_WITH_TIMEZONE_UPDATES = 54464; -type DBMS_MIN_REVISION_WITH_SPARSE_SERIALIZATION = 54465; -- ToDo -type DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION = 54466; -type DBMS_MIN_REVISION_WITH_TABLE_READ_ONLY_CHECK = 54467; -type DBMS_MIN_REVISION_WITH_SYSTEM_KEYWORDS_TABLE = 54468; -type DBMS_MIN_REVISION_WITH_ROWS_BEFORE_AGGREGATION = 54469; -type DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470; -type DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL = 54471; -type DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES = 54472; -type DBMS_MIN_REVISION_WITH_V2_DYNAMIC_AND_JSON_SERIALIZATION = 54473; -type DBMS_MIN_REVISION_WITH_SERVER_SETTINGS = 54474; -type DBMS_MIN_REVISON_WITH_JWT_IN_INTERSERVER = 54476; -type DBMS_MIN_REVISION_WITH_QUERY_PLAN_SERIALIZATION = 54477; -type DBMS_MIN_REVISON_WITH_PARALLEL_BLOCK_MARSHALLING = 54478; -type DBMS_MIN_REVISION_WITH_VERSIONED_CLUSTER_FUNCTION_PROTOCOL = 54479; -type DBMS_MIN_REVISION_WITH_OUT_OF_ORDER_BUCKETS_IN_AGGREGATION = 54480; -type DBMS_MIN_REVISION_WITH_COMPRESSED_LOGS_PROFILE_EVENTS_COLUMNS = 54481; +-- Internal +import ClickHaskell.Primitive.Serialization +import ClickHaskell.Primitive.TArray +import ClickHaskell.Primitive.TBool () +import ClickHaskell.Primitive.TDate (Date) +import ClickHaskell.Primitive.TDateTime (DateTime, DateTime64) +import ClickHaskell.Primitive.TDecimal +import ClickHaskell.Primitive.TEnum (Enum8, Enum16) +import ClickHaskell.Primitive.TFloat (Float32, Float64) +import ClickHaskell.Primitive.TInt () +import ClickHaskell.Primitive.TLowCardinality +import ClickHaskell.Primitive.TString (ChString) +import ClickHaskell.Primitive.TNullable (Nullable) +import ClickHaskell.Primitive.TUInt +import ClickHaskell.Primitive.TUUID (UUID) diff --git a/ClickHaskell/Primitive/Serialization.hs b/ClickHaskell/Primitive/Serialization.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/Serialization.hs @@ -0,0 +1,303 @@ +module ClickHaskell.Primitive.Serialization where + +-- GHC included +import Control.Applicative (liftA2) +import Control.DeepSeq (NFData) +import Data.Binary.Get +import Data.Bits (Bits (setBit, unsafeShiftL, unsafeShiftR, (.&.), (.|.)), xor) +import Data.ByteString.Builder +import Data.Int (Int64) +import Data.Type.Bool (Not) +import Data.Type.Equality (type (==)) +import Data.Typeable (Proxy (..)) +import Data.Word (Word64) +import GHC.Generics (C1, D1, Generic (..), K1 (K1), M1 (M1), Meta (MetaSel), Rec0, S1, type (:*:) (..)) +import GHC.TypeLits (KnownNat, Nat, natVal) +import Prelude hiding (liftA2) + + +------------------------------------------------------------------------------- +-- * Core serialization class +------------------------------------------------------------------------------- + +class Serializable chType + where + default serialize :: (Generic chType, GSerial (Rep chType)) => ProtocolRevision -> chType -> Builder + serialize :: ProtocolRevision -> chType -> Builder + serialize rev = gSerialize rev . from + + {-# INLINE deserialize #-} + default deserialize :: (Generic chType, GSerial (Rep chType)) => ProtocolRevision -> Get chType + deserialize :: ProtocolRevision -> Get chType + deserialize rev = to <$> gDeserialize rev + +{-# INLINE replicateGet #-} +replicateGet :: Serializable chType => ProtocolRevision -> UVarInt -> Get [chType] +replicateGet rev cnt0 = loopGet cnt0 + where + loopGet cnt + | cnt == 0 = pure [] + | otherwise = liftA2 (:) (deserialize rev) (loopGet (cnt - 1)) + +instance Serializable prim => Serializable [prim] where + serialize rev list + = serialize @UVarInt rev (fromIntegral $ Prelude.length list) + <> foldMap (serialize @prim rev) list + deserialize rev = do + len <- deserialize @UVarInt rev + replicateGet @prim rev len + {-# INLINE deserialize #-} + +instance Serializable () where + serialize _ () = "" + deserialize _ = pure () + + +class ToQueryPart chType where + toQueryPart :: chType -> Builder + + toQueryPartQuoted :: chType -> Builder + toQueryPartQuoted = (\part -> "'" <> part <> "'" ) . toQueryPart + +class IsChType chType + where + -- | Shows database original type name + -- + -- @ + -- chTypeName \@ChString = \"String\" + -- chTypeName \@(Nullable UInt32) = \"Nullable(UInt32)\" + -- @ + chTypeName :: String + + defaultValueOfTypeName :: chType + +class ToChType chType userType where + toChType :: userType -> chType + fromChType :: chType -> userType + +instance {-# OVERLAPPABLE #-} (IsChType chType, chType ~ inputType) => ToChType chType inputType where + toChType = id + fromChType = id + + +-- ** Generics + +class GSerial f where + gSerialize :: ProtocolRevision -> f p -> Builder + gDeserialize :: ProtocolRevision -> Get (f p) + +instance GSerial f => GSerial (D1 c (C1 c2 f)) where + gSerialize rev (M1 (M1 re)) = gSerialize rev re + {-# INLINE gSerialize #-} + gDeserialize rev = M1 . M1 <$> gDeserialize rev + {-# INLINE gDeserialize #-} + +instance (GSerial left1, GSerial right) => GSerial (left1 :*: right) where + gSerialize rev (l :*: r) = gSerialize rev l <> gSerialize rev r + {-# INLINE gSerialize #-} + gDeserialize rev = do + liftA2 (:*:) + (gDeserialize rev) + (gDeserialize rev) + {-# INLINE gDeserialize #-} + +instance + (Serializable chType, Not (sel == "server_revision") ~ True) + => + GSerial (S1 ('MetaSel ('Just sel) a b c) (Rec0 chType)) where + gSerialize rev (M1 (K1 re)) = serialize rev re + {-# INLINE gSerialize #-} + gDeserialize rev = M1 . K1 <$> deserialize @chType rev + {-# INLINE gDeserialize #-} + + +instance {-# OVERLAPPING #-} + GSerial right + => + GSerial (S1 ('MetaSel ('Just "server_revision") a b c) (Rec0 ProtocolRevision) :*: right) + where + gSerialize rev (M1 (K1 (MkProtocolRevision server_rev)) :*: right)= do + serialize rev server_rev <> gSerialize rev right + {-# INLINE gSerialize #-} + gDeserialize rev = do + chosenRev <- min rev . MkProtocolRevision <$> deserialize @UVarInt rev + liftA2 (:*:) + (pure . M1 . K1 $ chosenRev) + (gDeserialize @right chosenRev) + {-# INLINE gDeserialize #-} + + + + +------------------------------------------------------------------------------- +-- * Protocol parts +------------------------------------------------------------------------------- + +-- ** UVarInt + +{- | + Unsigned variable-length quantity encoding + + Part of protocol implementation +-} +newtype UVarInt = MkUVarInt Word64 + deriving newtype (Show, Eq, Num, Bits, Enum, Ord, Real, Integral, Bounded, NFData) + +instance Serializable UVarInt where + serialize _ = goUVarIntSer + where + goUVarIntSer i + | i < 0x80 = word8 (fromIntegral i) + | otherwise = word8 (setBit (fromIntegral i) 7) <> goUVarIntSer (i `unsafeShiftR` 7) + deserialize _ = goUVarIntDeser 0 (0 :: UVarInt) + where + goUVarIntDeser i o | i < 10 = do + byte <- getWord8 + let o' = o .|. ((fromIntegral byte .&. 0x7f) `unsafeShiftL` (7 * i)) + if byte .&. 0x80 == 0 then pure $! o' else goUVarIntDeser (i + 1) $! o' + goUVarIntDeser _ _ = fail "input exceeds varuint size" + {-# INLINE deserialize #-} + +newtype VarInt = MkVarInt Int64 + deriving newtype (Show, Eq, Num, Bits, Enum, Ord, Real, Integral, Bounded, NFData) + +instance Serializable VarInt where + serialize rev (MkVarInt int64) = + serialize rev (MkUVarInt (zigZagEncode int64)) + where + zigZagEncode :: Int64 -> Word64 + zigZagEncode i = fromIntegral ((i `unsafeShiftL` 1) `xor` (i `unsafeShiftR` 63)) + {-# INLINE zigZagEncode #-} + + {-# INLINE deserialize #-} + deserialize rev = do + MkUVarInt u <- deserialize rev + pure $! MkVarInt (zigZagDecode u) + where + zigZagDecode :: Word64 -> Int64 + zigZagDecode u = + fromIntegral ((u `unsafeShiftR` 1) `xor` negate (u .&. 1)) + {-# INLINE zigZagDecode #-} + +-- >>> show major <> "." <> show minor <> "." <> show patch +-- "1.0.0" +major, minor, patch :: UVarInt +major = 1 +minor = 1 +patch = 0 + +newtype ProtocolRevision = MkProtocolRevision UVarInt + deriving newtype (Eq, Num, Ord, Serializable) + +mkRev :: forall nat . KnownNat nat => ProtocolRevision +mkRev = (fromIntegral . natVal) (Proxy @nat) + + +-- ** Versioning + +{- | +Protocol implementation part for backward compatilibity formalization + +NB: + + Be carefull with `BeforeRevision` value. + + If **chosen protocol** revision would be >= **revisionNumber** + then you would get an exception during serialization. + + To avoid this: + + 1. On client side - provide `AfterRevision` with some empty value + 2. On proxy side - provide server-to-server packets mapping with fallbacks on revision upgrade + +-} +data Revisioned (revisionNumber :: Nat) b a = BeforeRevision b | AfterRevision a + +type SinceRevision after rev = Revisioned rev () after + +instance + (KnownNat revision, Serializable before, Serializable after) + => + Serializable (Revisioned revision before after) + where + serialize rev sinceRevVal = + if rev < mkRev @revision + then mempty + else case sinceRevVal of + BeforeRevision _b -> error "Protocol-specific implementation error" -- Watch `Revisioned` note + AfterRevision a -> serialize rev a + deserialize rev = + if rev < mkRev @revision + then BeforeRevision <$> deserialize @before rev + else AfterRevision <$> deserialize @after rev + + +{- + Slightly modified C++ sources: + https://github.com/ClickHouse/ClickHouse/blob/eb4a74d7412a1fcf52727cd8b00b365d6b9ed86c/src/Core/ProtocolDefines.h#L6 +-} +type DBMS_TCP_PROTOCOL_VERSION = DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS; + +type DBMS_MIN_REVISION_WITH_CLIENT_INFO = 54032; +type DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE = 54058; +type DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO = 54060; +type DBMS_MIN_REVISION_WITH_TABLES_STATUS = 54226; +type DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE = 54337; +type DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME = 54372; +type DBMS_MIN_REVISION_WITH_VERSION_PATCH = 54401; +type DBMS_MIN_REVISION_WITH_SERVER_LOGS = 54406; +type DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD = 54448; +type DBMS_MIN_MAJOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD = 21; +type DBMS_MIN_MINOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD = 4; +type DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA = 54410; +type DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE = 54405; +type DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO = 54420; +type DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS = 54429; +type DBMS_MIN_REVISION_WITH_SCALARS = 54429; +type DBMS_MIN_REVISION_WITH_OPENTELEMETRY = 54442; +type DBMS_MIN_REVISION_WITH_AGGREGATE_FUNCTIONS_VERSIONING = 54452; +type DBMS_CLUSTER_INITIAL_PROCESSING_PROTOCOL_VERSION = 1; +type DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA = 2; +type DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = 2; +type DATA_LAKE_TABLE_STATE_SNAPSHOT_PROTOCOL_VERSION = 1; +type DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3; +type DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_MARK_SEGMENT_SIZE_FIELD = 4; +type DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_PROJECTION = 5; +type DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 5; +type DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453; +type DBMS_MIN_REVISION_WITH_QUERY_AND_LINE_NUMBERS = 54475; +type DBMS_MERGE_TREE_PART_INFO_VERSION = 1; +type DBMS_QUERY_PLAN_SERIALIZATION_VERSION = 0; +type DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET = 54441; +type DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO = 54443; +type DBMS_MIN_REVISION_WITH_REFERER_IN_CLIENT_INFO = 54447; +type DBMS_MIN_PROTOCOL_VERSION_WITH_DISTRIBUTED_DEPTH = 54448; +type DBMS_MIN_PROTOCOL_VERSION_WITH_INCREMENTAL_PROFILE_EVENTS = 54451; +type DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION = 54454; +type DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME = 54449; +type DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT = 54456; +type DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED = 54457; +type DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM = 54458; +type DBMS_MIN_PROTOCOL_VERSION_WITH_QUOTA_KEY = 54458; +type DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS = 54459; +type DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS = 54460; +type DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES = 54461; +type DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 = 54462; +type DBMS_MIN_PROTOCOL_VERSION_WITH_TOTAL_BYTES_IN_PROGRESS = 54463; +type DBMS_MIN_PROTOCOL_VERSION_WITH_TIMEZONE_UPDATES = 54464; +type DBMS_MIN_REVISION_WITH_SPARSE_SERIALIZATION = 54465; -- ToDo +type DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION = 54466; +type DBMS_MIN_REVISION_WITH_TABLE_READ_ONLY_CHECK = 54467; +type DBMS_MIN_REVISION_WITH_SYSTEM_KEYWORDS_TABLE = 54468; +type DBMS_MIN_REVISION_WITH_ROWS_BEFORE_AGGREGATION = 54469; +type DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470; +type DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL = 54471; +type DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES = 54472; +type DBMS_MIN_REVISION_WITH_V2_DYNAMIC_AND_JSON_SERIALIZATION = 54473; +type DBMS_MIN_REVISION_WITH_SERVER_SETTINGS = 54474; +type DBMS_MIN_REVISON_WITH_JWT_IN_INTERSERVER = 54476; +type DBMS_MIN_REVISION_WITH_QUERY_PLAN_SERIALIZATION = 54477; +type DBMS_MIN_REVISON_WITH_PARALLEL_BLOCK_MARSHALLING = 54478; +type DBMS_MIN_REVISION_WITH_VERSIONED_CLUSTER_FUNCTION_PROTOCOL = 54479; +type DBMS_MIN_REVISION_WITH_OUT_OF_ORDER_BUCKETS_IN_AGGREGATION = 54480; +type DBMS_MIN_REVISION_WITH_COMPRESSED_LOGS_PROFILE_EVENTS_COLUMNS = 54481; diff --git a/ClickHaskell/Primitive/TArray.hs b/ClickHaskell/Primitive/TArray.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TArray.hs @@ -0,0 +1,40 @@ +module ClickHaskell.Primitive.TArray where + +-- Internal +import ClickHaskell.Primitive.Serialization +import ClickHaskell.Primitive.TString (escapeString) + +-- GHC included +import Control.DeepSeq (NFData) +import Data.ByteString (toStrict) +import Data.ByteString.Builder (toLazyByteString) +import Data.Coerce (coerce) +import Data.List (uncons) + + +-- * Array + +-- | ClickHouse Array column type +newtype Array a = MkChArray [a] + deriving newtype (Show, Eq, NFData, Foldable) + +instance IsChType chType => IsChType (Array chType) + where + chTypeName = "Array(" <> chTypeName @chType <> ")" + defaultValueOfTypeName = MkChArray [] + +instance ToChType chType inputType => ToChType (Array chType) [inputType] + where + toChType = MkChArray . map toChType + fromChType (MkChArray values) = map fromChType values + +instance (IsChType chType, ToQueryPart chType) => ToQueryPart (Array chType) + where + toQueryPart + = (\x -> "[" <> x <> "]") + . (maybe "" (uncurry (foldl (\a b -> a <> "," <> b))) . uncons + . map (toQueryPart @chType)) . coerce @(Array chType) @[chType] + toQueryPartQuoted + = (\x -> "'[" <> escapeString (toStrict $ toLazyByteString x) <> "]'") + . (maybe "" (uncurry (foldl (\a b -> a <> "," <> b))) . uncons + . map (toQueryPart @chType)) . coerce @(Array chType) @[chType] diff --git a/ClickHaskell/Primitive/TBool.hs b/ClickHaskell/Primitive/TBool.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TBool.hs @@ -0,0 +1,25 @@ +{-# OPTIONS_GHC -Wno-orphans #-} +module ClickHaskell.Primitive.TBool where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Data.ByteString.Builder (int8) +import Data.Bool ( bool ) + +-- External +import Data.Binary.Get (getInt8) + + +instance IsChType Bool where + chTypeName = "Bool" + defaultValueOfTypeName = False + +instance Serializable Bool where + serialize _ = int8 . bool 0 1 + deserialize _ = (\int -> case int of 0->False; _->True) <$> getInt8 + {-# INLINE deserialize #-} + +instance ToQueryPart Bool where + toQueryPart = bool "false" "true" diff --git a/ClickHaskell/Primitive/TDate.hs b/ClickHaskell/Primitive/TDate.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TDate.hs @@ -0,0 +1,31 @@ +module ClickHaskell.Primitive.TDate where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Data.ByteString.Builder (word16LE) +import Data.Word (Word16) +import Data.Bits (Bits) +import Control.DeepSeq (NFData) + +-- External +import Data.Binary.Get (getWord16le) + + +{- | ClickHouse Date column type -} +newtype Date = MkDate Word16 + deriving newtype (Show, Eq, Bits, Bounded, Enum, NFData, Num) + +instance IsChType Date where + chTypeName = "Date" + defaultValueOfTypeName = 0 + +instance Serializable Date where + serialize _ (MkDate w16) = word16LE w16 + deserialize _ = MkDate <$> getWord16le + {-# INLINE deserialize #-} + +instance ToChType Date Word16 where + toChType = MkDate + fromChType (MkDate w16) = w16 diff --git a/ClickHaskell/Primitive/TDateTime.hs b/ClickHaskell/Primitive/TDateTime.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TDateTime.hs @@ -0,0 +1,123 @@ +module ClickHaskell.Primitive.TDateTime where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Control.DeepSeq (NFData) +import Data.Bits (Bits) +import Data.ByteString.Builder (byteString, word32LE, word64LE) +import Data.ByteString.Char8 as BS8 (length, pack, replicate) +import Data.Coerce (coerce) +import Data.Proxy (Proxy (..)) +import Data.Time (UTCTime) +import Data.Time.Clock.POSIX (posixSecondsToUTCTime, utcTimeToPOSIXSeconds) +import Data.Word (Word32, Word64) +import GHC.TypeLits (KnownNat, KnownSymbol, Nat, Symbol, natVal, symbolVal) + +-- External +import Data.Binary.Get (getWord32le, getWord64le) + +-- * DateTime + +{- | +ClickHouse DateTime column type (parametrized with timezone) + +>>> chTypeName @(DateTime "") +"DateTime" +>>> chTypeName @(DateTime "UTC") +"DateTime('UTC')" + +__Note:__ 'DateTime' stores whole seconds only, so converting from 'UTCTime' \ +will drop any sub-second precision. + +>>> let myUtcTime = posixSecondsToUTCTime 0.042_042 +>>> toChType @(DateTime "") @UTCTime myUtcTime +0 +-} +newtype DateTime (tz :: Symbol) = MkDateTime Word32 + deriving newtype (Show, Eq, Num, Bits, Enum, Ord, Real, Integral, Bounded, NFData) + +instance KnownSymbol tz => IsChType (DateTime tz) + where + chTypeName = case symbolVal @tz Proxy of + "" -> "DateTime" + tz -> "DateTime('" <> tz <> "')" + defaultValueOfTypeName = MkDateTime 0 + +instance Serializable (DateTime tz) where + serialize _ (MkDateTime w32) = word32LE w32 + deserialize _ = MkDateTime <$> getWord32le + {-# INLINE deserialize #-} + +instance ToChType (DateTime tz) Word32 where + toChType = MkDateTime + fromChType (MkDateTime w32)= w32 + +instance ToChType (DateTime tz) UTCTime where + toChType = MkDateTime . floor . utcTimeToPOSIXSeconds + fromChType (MkDateTime w32) = posixSecondsToUTCTime (fromIntegral w32) + +instance ToQueryPart (DateTime tz) + where + toQueryPart chDateTime = let time = BS8.pack . show . coerce @(DateTime tz) @Word32 $ chDateTime + in byteString (BS8.replicate (10 - BS8.length time) '0' <> time) + + +-- * DateTime64 + +{- | +ClickHouse DateTime64 column type (parametrized with timezone) + +>>> chTypeName @(DateTime64 3 "") +"DateTime64(3)" +>>> chTypeName @(DateTime64 3 "UTC") +"DateTime64(3, 'UTC')" + +__Note:__ conversion from 'UTCTime' may lose sub-second precision if \ +the @precision@ parameter is lower than the actual timestamp precision. + +>>> let myUtcTime = posixSecondsToUTCTime 42.000_000_042 +>>> toChType @(DateTime64 6 "") @UTCTime myUtcTime +42000000 +>>> toChType @(DateTime64 9 "") @UTCTime myUtcTime +42000000042 +-} +newtype DateTime64 (precision :: Nat) (tz :: Symbol) = MkDateTime64 Word64 + deriving newtype (Show, Eq, Num, Bits, Enum, Ord, Real, Integral, Bounded, NFData) + +instance + (KnownSymbol tz, KnownNat precision) + => + IsChType (DateTime64 precision tz) + where + chTypeName = + let + prec = show (natVal @precision Proxy) + in + case symbolVal @tz Proxy of + "" -> "DateTime64(" <> prec <> ")" + tz -> "DateTime64(" <> prec <> ", '" <> tz <> "')" + defaultValueOfTypeName = MkDateTime64 0 + +instance Serializable (DateTime64 precision tz) where + serialize _ (MkDateTime64 w64) = word64LE w64 + deserialize _ = MkDateTime64 <$> getWord64le + {-# INLINE deserialize #-} + +instance ToChType (DateTime64 precision tz) Word64 where + toChType = MkDateTime64 + fromChType (MkDateTime64 w64) = w64 + +instance KnownNat precision => ToChType (DateTime64 precision tz) UTCTime where + toChType = MkDateTime64 . floor . (* (10 ^ natVal (Proxy @precision))) + . utcTimeToPOSIXSeconds + fromChType (MkDateTime64 w64) = posixSecondsToUTCTime + $ (/ (10 ^ natVal (Proxy @precision))) $ fromIntegral w64 + +-- ToDo: Need to be fixed +-- instance ToQueryPart (DateTime64 precision tz) +-- where +-- toQueryPart chDateTime = +-- let time = BS8.pack . show . fromChType @_ @Word64 $ chDateTime +-- in byteString (BS8.replicate (12 - BS8.length time) '0' <> time) diff --git a/ClickHaskell/Primitive/TDecimal.hs b/ClickHaskell/Primitive/TDecimal.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TDecimal.hs @@ -0,0 +1,237 @@ +module ClickHaskell.Primitive.TDecimal where + +-- Internal +import ClickHaskell.Primitive.Serialization +import ClickHaskell.Primitive.TInt (Int64, Int32) + +-- GHC included +import Data.Binary.Get +import Data.ByteString.Builder +import Data.Fixed (Fixed (..)) +import Data.Kind (Constraint) +import Data.Type.Bool +import Data.Type.Ord +import Data.Typeable (Proxy (..)) +import GHC.TypeLits (ErrorMessage (..), KnownNat, Nat, TypeError, natVal, type (^)) + +-- External +import Data.WideWord (Int128 (..), Int256) +import Data.ByteString.Char8 as BS8 (pack) + + +-- ** Decimal32 + +{- | +>>> chTypeName @(Decimal32 9 1) +"Decimal(9, 1)" +>>> toChType @(Decimal32 1 1) @(Fixed (10^1)) 100000.1 +100000.1 +>>> toChType @(Decimal32 9 5) @(Fixed (10^5)) (10^4 + 0.1) +10000.10000 + +Note: there are no overflow checks + +>>> toChType @(Decimal32 9 5) @(Fixed (10^5)) (10^5 + 0.1) +14100.75408 +-} +newtype Decimal32 (p :: Nat) (s :: Nat) = MkDecimal32 Int32 + +instance KnownNat (10^s) => Show (Decimal32 p s) where + show (MkDecimal32 int32) = show $ MkFixed @_ @(10^s)(fromIntegral int32) +deriving newtype instance KnownNat (10^s) => Eq (Decimal32 p s) +deriving newtype instance KnownNat (10^s) => Ord (Decimal32 p s) + +instance + (ValidRanges 32 1 9 p s, KnownNat p, KnownNat s, KnownNat (10 ^ s)) + => + IsChType (Decimal32 p s) where + chTypeName = + let p = show (natVal @p Proxy) + s = show (natVal @s Proxy) + in "Decimal(" <> p <> ", "<> s <> ")" + defaultValueOfTypeName = MkDecimal32 0 + +instance KnownNat (10 ^ s) => Serializable (Decimal32 p s) where + serialize _ (MkDecimal32 int) = int32LE int + deserialize _ = MkDecimal32 <$> getInt32le + {-# INLINE deserialize #-} + +instance + ( sPowered ~ 10^s + , IsChType (Decimal32 p s) + ) => + ToChType (Decimal32 p s) (Fixed sPowered) where + toChType (MkFixed fixedRep) = MkDecimal32 (fromIntegral fixedRep) + fromChType (MkDecimal32 int32) = MkFixed (fromIntegral int32) + +instance KnownNat (10^s) => ToQueryPart (Decimal32 p s) where + toQueryPart dec = byteString (BS8.pack $ show dec) + + +-- ** Decimal64 + +{- | +>>> chTypeName @(Decimal64 10 1) +"Decimal(10, 1)" +>>> toChType @(Decimal64 10 5) @(Fixed (10^5)) (10^13 + 0.1) +10000000000000.10000 + +Note: there are no overflow checks + +>>> toChType @(Decimal64 10 5) @(Fixed (10^5)) (10^14 + 0.1) +-84467440737095.41616 +-} +newtype Decimal64 (p :: Nat) (s :: Nat) = MkDecimal64 Int64 + +instance KnownNat (10^s) => Show (Decimal64 p s) where + show (MkDecimal64 int64) = show $ MkFixed @_ @(10^s)(fromIntegral int64) +deriving newtype instance KnownNat (10^s) => Eq (Decimal64 p s) +deriving newtype instance KnownNat (10^s) => Ord (Decimal64 p s) + +instance + (ValidRanges 64 10 18 p s, KnownNat p, KnownNat s, KnownNat (10 ^ s)) + => + IsChType (Decimal64 p s) where + chTypeName = + let p = show (natVal @p Proxy) + s = show (natVal @s Proxy) + in "Decimal(" <> p <> ", "<> s <> ")" + defaultValueOfTypeName = MkDecimal64 0 + +instance KnownNat (10 ^ s) => Serializable (Decimal64 p s) where + serialize _ (MkDecimal64 int) = int64LE int + deserialize _ = MkDecimal64 <$> getInt64le + {-# INLINE deserialize #-} + +instance + ( sPowered ~ 10^s + , IsChType (Decimal64 p s) + ) => + ToChType (Decimal64 p s) (Fixed sPowered) where + toChType (MkFixed fixedRep) = MkDecimal64 (fromIntegral fixedRep) + fromChType (MkDecimal64 int64) = MkFixed (fromIntegral int64) + +instance KnownNat (10^s) => ToQueryPart (Decimal64 p s) where + toQueryPart dec = byteString (BS8.pack $ show dec) + + +-- ** Decimal128 + +{- | +>>> chTypeName @(Decimal128 19 1) +"Decimal(19, 1)" +>>> toChType @(Decimal128 19 5) @(Fixed (10^5)) (10^33 + 0.1) +1000000000000000000000000000000000.10000 + +Note: there are no overflow checks + +>>> toChType @(Decimal128 19 5) @(Fixed (10^5)) (10^34 + 0.1) +-208471007628153903901238222953046.24368 +-} +newtype Decimal128 (p :: Nat) (s :: Nat) = MkDecimal128 Int128 + +instance KnownNat (10^s) => Show (Decimal128 p s) where + show (MkDecimal128 int128) = show $ MkFixed @_ @(10^s)(fromIntegral int128) +deriving newtype instance KnownNat (10^s) => Eq (Decimal128 p s) +deriving newtype instance KnownNat (10^s) => Ord (Decimal128 p s) + +instance + (ValidRanges 128 19 38 p s, KnownNat p, KnownNat s, KnownNat (10 ^ s)) + => + IsChType (Decimal128 p s) where + chTypeName = + let p = show (natVal @p Proxy) + s = show (natVal @s Proxy) + in "Decimal(" <> p <> ", "<> s <> ")" + defaultValueOfTypeName = MkDecimal128 0 + +instance Serializable (Decimal128 p s) where + serialize rev (MkDecimal128 int) = serialize @Int128 rev int + deserialize rev = MkDecimal128 <$> deserialize @Int128 rev + {-# INLINE deserialize #-} + +instance + ( sPowered ~ 10^s + , IsChType (Decimal128 p s) + ) => + ToChType (Decimal128 p s) (Fixed sPowered) where + toChType (MkFixed fixedRep) = MkDecimal128 (fromIntegral fixedRep) + fromChType (MkDecimal128 int128) = MkFixed (fromIntegral int128) + +instance KnownNat (10^s) => ToQueryPart (Decimal128 p s) where + toQueryPart dec = byteString (BS8.pack $ show dec) + + +-- ** Decimal256 + +{- | +>>> chTypeName @(Decimal256 39 1) +"Decimal(39, 1)" +>>> toChType @(Decimal256 39 1) @(Fixed (10^1)) (10^71 + 0.1) +100000000000000000000000000000000000000000000000000000000000000000000000.1 + +Note: there are no overflow checks + +>>> toChType @(Decimal256 39 5) @(Fixed (10^5)) (10^72 + 0.1) +-157920892373161954235709850086879078532699846656405640394575840079131296.29936 +-} +newtype Decimal256 (p :: Nat) (s :: Nat) = MkDecimal256 Int256 + +instance KnownNat (10^s) => Show (Decimal256 p s) where + show (MkDecimal256 int256) = show $ MkFixed @_ @(10^s)(fromIntegral int256) +deriving newtype instance KnownNat (10^s) => Eq (Decimal256 p s) +deriving newtype instance KnownNat (10^s) => Ord (Decimal256 p s) + +instance + (ValidRanges 256 39 76 p s, KnownNat p, KnownNat s, KnownNat (10 ^ s)) + => + IsChType (Decimal256 p s) where + chTypeName = + let p = show (natVal @p Proxy) + s = show (natVal @s Proxy) + in "Decimal(" <> p <> ", "<> s <> ")" + defaultValueOfTypeName = MkDecimal256 0 + +instance Serializable (Decimal256 p s) where + serialize rev (MkDecimal256 int) = serialize @Int256 rev int + deserialize rev = MkDecimal256 <$> deserialize @Int256 rev + {-# INLINE deserialize #-} + +instance + ( sPowered ~ 10^s + , IsChType (Decimal256 p s) + ) => + ToChType (Decimal256 p s) (Fixed sPowered) where + toChType (MkFixed fixedRep) = MkDecimal256 (fromIntegral fixedRep) + fromChType (MkDecimal256 int256) = MkFixed (fromIntegral int256) + +instance KnownNat (10^s) => ToQueryPart (Decimal256 p s) where + toQueryPart dec = byteString (BS8.pack $ show dec) + + +-- Range validations + +type family ValidRanges (size :: Nat) (pMin :: Nat) (pMax :: Nat) (p :: Nat) (s :: Nat) :: Constraint + where + ValidRanges size pMin pMax p s = + If (p >=? 0 && s <=? p) + ( + If + (pMin <=? p && p <=? pMax) + (() :: Constraint) + (TypeError + ( 'Text "Precision (p=" :<>: ShowType p :<>: 'Text ") should satisfy " + :<>: ShowType pMin :<>: 'Text " <= p <= " :<>: ShowType pMax + :<>: 'Text " for " :<>: DecimalType size + ) + ) + ) + (TypeError + ( 'Text "Scale (s=" :<>: ShowType s :<>: 'Text ") and " + :<>: 'Text "precision (p=" :<>: ShowType p :<>: 'Text ") " + :<>: 'Text "should satisfy 0 <= s <= p for " + :<>: DecimalType size + ) + ) + +type DecimalType (size :: Nat) = 'Text "Decimal" :<>: ShowType size :<>: 'Text " type" diff --git a/ClickHaskell/Primitive/TEnum.hs b/ClickHaskell/Primitive/TEnum.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TEnum.hs @@ -0,0 +1,61 @@ +module ClickHaskell.Primitive.TEnum where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Data.Bits (Bits) +import Data.ByteString.Builder (byteString, int8, int16LE) +import Data.ByteString.Char8 as BS8 (pack) +import Data.Int (Int8, Int16) +import Data.Typeable (Proxy (..)) +import GHC.TypeLits (KnownSymbol, Symbol, symbolVal) + +-- External +import Data.Binary.Get (getInt8, getInt16le) + + +-- * Enum8 + +newtype Enum8 (enums :: Symbol) = MkEnum8 Int8 + deriving newtype (Show, Eq, Num, Bits, Bounded, Enum) + +instance KnownSymbol enums => IsChType (Enum8 enums) where + chTypeName = "Enum8(" <> symbolVal @enums Proxy <> ")" + defaultValueOfTypeName = 0 + +instance Serializable (Enum8 enums) where + serialize _ (MkEnum8 i8) = int8 i8 + deserialize _ = MkEnum8 <$> getInt8 + {-# INLINE deserialize #-} + +instance ToChType (Enum8 enums) Int8 where + toChType = MkEnum8 + fromChType (MkEnum8 i8)= i8 + +instance ToQueryPart (Enum8 enums) where + toQueryPart = byteString . BS8.pack . show . fromChType @_ @Int8 + + +-- * Enum16 + +newtype Enum16 (enums :: Symbol) = MkEnum16 Int16 + deriving newtype (Show, Eq, Num, Bits, Bounded, Enum) + +instance KnownSymbol enums => IsChType (Enum16 enums) where + chTypeName = "Enum16(" <> symbolVal @enums Proxy <> ")" + defaultValueOfTypeName = 0 + +instance Serializable (Enum16 enums) where + serialize _ (MkEnum16 i16) = int16LE i16 + deserialize _ = MkEnum16 <$> getInt16le + {-# INLINE deserialize #-} + +instance ToChType (Enum16 enums) Int16 where + toChType = MkEnum16 + fromChType (MkEnum16 i16) = i16 + +instance ToQueryPart (Enum16 enums) where + toQueryPart = byteString . BS8.pack . show . fromChType @_ @Int16 + + diff --git a/ClickHaskell/Primitive/TFloat.hs b/ClickHaskell/Primitive/TFloat.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TFloat.hs @@ -0,0 +1,49 @@ +{-# OPTIONS_GHC -Wno-orphans #-} +module ClickHaskell.Primitive.TFloat where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Data.Binary.Get +import Data.ByteString.Builder +import Data.ByteString.Char8 as BS8 (pack) +import Prelude hiding (liftA2) + +-- External +import Data.Binary.Put (putFloatle, putDoublele, execPut) + + +-- * Float32 + +type Float32 = Float + +instance IsChType Float32 where + chTypeName = "Float32" + defaultValueOfTypeName = 0 + +instance Serializable Float32 where + serialize _ f32 = (execPut . putFloatle) f32 + deserialize _ = getFloatle + {-# INLINE deserialize #-} + +instance ToQueryPart Float32 where + toQueryPart = byteString . BS8.pack . show + + + +-- * Float64 + +type Float64 = Double + +instance IsChType Float64 where + chTypeName = "Float64" + defaultValueOfTypeName = 0 + +instance Serializable Float64 where + serialize _ f64 = (execPut . putDoublele) f64 + deserialize _ = getDoublele + {-# INLINE deserialize #-} + +instance ToQueryPart Float64 where + toQueryPart = byteString . BS8.pack . show diff --git a/ClickHaskell/Primitive/TInt.hs b/ClickHaskell/Primitive/TInt.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TInt.hs @@ -0,0 +1,123 @@ +{-# OPTIONS_GHC -Wno-orphans #-} +module ClickHaskell.Primitive.TInt + ( Int8, Int16, Int32, Int64, Int128, Int256 + ) where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Data.Int (Int8, Int16, Int32, Int64) +import Data.ByteString.Builder (byteString, int8, int16LE, int32LE, int64LE, word64LE) +import Data.ByteString.Char8 as BS8 (pack) + +-- External +import Data.Binary.Get (getInt8, getInt16le, getInt32le, getInt64le, getWord64le) +import Data.WideWord + + +instance IsChType Int8 where + chTypeName = "Int8" + defaultValueOfTypeName = 0 + +instance Serializable Int8 where + serialize _ = int8 + deserialize _ = getInt8 + {-# INLINE deserialize #-} + +instance ToQueryPart Int8 where + toQueryPart = byteString . BS8.pack . show + + +-- * Int16 + +instance IsChType Int16 where + chTypeName = "Int16" + defaultValueOfTypeName = 0 + +instance Serializable Int16 where + serialize _ = int16LE + deserialize _ = getInt16le + {-# INLINE deserialize #-} + +instance ToQueryPart Int16 where + toQueryPart = byteString . BS8.pack . show + + +-- * Int32 + +instance IsChType Int32 where + chTypeName = "Int32" + defaultValueOfTypeName = 0 + +instance Serializable Int32 where + serialize _ = int32LE + deserialize _ = getInt32le + {-# INLINE deserialize #-} + +instance ToQueryPart Int32 where + toQueryPart = byteString . BS8.pack . show + + + + + + + +instance IsChType Int64 where + chTypeName = "Int64" + defaultValueOfTypeName = 0 + +instance Serializable Int64 where + serialize _ = int64LE + deserialize _ = getInt64le + {-# INLINE deserialize #-} + +instance ToQueryPart Int64 where + toQueryPart = byteString . BS8.pack . show + + + + + + + + +instance IsChType Int128 where + chTypeName = "Int128" + defaultValueOfTypeName = 0 + +instance Serializable Int128 where + serialize _ = (\(Int128 hi lo) -> word64LE lo <> word64LE hi) + deserialize _ = do + low <- getWord64le + high <- getWord64le + pure $ Int128 high low + {-# INLINE deserialize #-} + +instance ToQueryPart Int128 where + toQueryPart = byteString . BS8.pack . show + + + + + + + + +instance IsChType Int256 where + chTypeName = "Int256" + defaultValueOfTypeName = 0 + +instance Serializable Int256 where + serialize _ = (\(Int256 a1 a2 a3 a4) -> word64LE a4 <> word64LE a3 <> word64LE a2 <> word64LE a1) + deserialize _ = do + low <- getWord64le + mid1 <- getWord64le + mid2 <- getWord64le + high <- getWord64le + pure $ Int256 high mid2 mid1 low + {-# INLINE deserialize #-} + +instance ToQueryPart Int256 where + toQueryPart = byteString . BS8.pack . show diff --git a/ClickHaskell/Primitive/TLowCardinality.hs b/ClickHaskell/Primitive/TLowCardinality.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TLowCardinality.hs @@ -0,0 +1,55 @@ +module ClickHaskell.Primitive.TLowCardinality where + +-- Internal +import ClickHaskell.Primitive.Serialization +import ClickHaskell.Primitive.TString +import ClickHaskell.Primitive.TNullable + +-- GHC included +import Control.DeepSeq (NFData) +import Data.String (IsString (..)) +import GHC.TypeLits (ErrorMessage (..), TypeError) + + +-- * LowCardinality + +-- | ClickHouse LowCardinality(T) column type +newtype LowCardinality chType = MkLowCardinality chType +instance IsLowCardinalitySupported chType => IsChType (LowCardinality chType) + where + chTypeName = "LowCardinality(" <> chTypeName @chType <> ")" + defaultValueOfTypeName = MkLowCardinality $ defaultValueOfTypeName @chType + +deriving newtype instance (Eq chType, IsLowCardinalitySupported chType) => Eq (LowCardinality chType) +deriving newtype instance (NFData chType, IsLowCardinalitySupported chType) => NFData (LowCardinality chType) +deriving newtype instance IsString (LowCardinality ChString) + +class IsChType chType => IsLowCardinalitySupported chType +instance IsLowCardinalitySupported ChString +instance + ( IsLowCardinalitySupported chType + , IsChType (Nullable chType) + ) => + IsLowCardinalitySupported (Nullable chType) + +instance {-# OVERLAPPABLE #-} + ( IsChType chType + , TypeError + ( 'Text "LowCardinality(" ':<>: 'ShowType chType ':<>: 'Text ") is unsupported" + ':$$: 'Text "Use one of these types:" + ':$$: 'Text " ChString" + ':$$: 'Text " DateTime" + ':$$: 'Text " Nullable(T)" + ) + ) => IsLowCardinalitySupported chType + +instance + ToChType inputType chType + => + ToChType (LowCardinality inputType) chType where + toChType = MkLowCardinality . toChType + fromChType (MkLowCardinality lc)= fromChType @inputType lc + +instance ToQueryPart chType => ToQueryPart (LowCardinality chType) + where + toQueryPart (MkLowCardinality chType) = toQueryPart chType diff --git a/ClickHaskell/Primitive/TNullable.hs b/ClickHaskell/Primitive/TNullable.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TNullable.hs @@ -0,0 +1,28 @@ +{-# OPTIONS_GHC -Wno-orphans #-} +module ClickHaskell.Primitive.TNullable where + +-- Internal +import ClickHaskell.Primitive.Serialization + + +{- | ClickHouse Nullable(T) column type + (type synonym for Maybe) + -} +type Nullable = Maybe +instance IsChType chType => IsChType (Nullable chType) + where + chTypeName = "Nullable(" <> chTypeName @chType <> ")" + defaultValueOfTypeName = Nothing + +instance + ToChType inputType chType + => + ToChType (Nullable inputType) (Nullable chType) + where + toChType = fmap (toChType @inputType @chType) + fromChType = fmap (fromChType @inputType) + +instance ToQueryPart chType => ToQueryPart (Nullable chType) + where + toQueryPart = maybe "null" toQueryPart + diff --git a/ClickHaskell/Primitive/TString.hs b/ClickHaskell/Primitive/TString.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TString.hs @@ -0,0 +1,60 @@ + + +module ClickHaskell.Primitive.TString where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Control.DeepSeq (NFData) +import Data.Binary.Get +import Data.ByteString as BS (ByteString, length) +import Data.ByteString.Builder +import Data.ByteString.Char8 as BS8 (pack, unpack, concatMap, singleton) +import Data.ByteString.Lazy (toStrict) +import Data.String (IsString (..)) +import Prelude hiding (liftA2) + + + +-- ** ChString + +{- | ClickHouse String column type -} +newtype ChString = MkChString BS.ByteString + deriving newtype (Show, Eq, IsString, NFData) + +instance IsChType ChString where + chTypeName = "String" + defaultValueOfTypeName = "" + +instance Serializable ChString where + serialize rev (MkChString str) = (serialize @UVarInt rev . fromIntegral . BS.length) str <> byteString str + deserialize rev = do + len <- deserialize @UVarInt rev + MkChString <$> (getByteString . fromIntegral) len + {-# INLINE deserialize #-} + +instance ToChType ChString BS.ByteString where + toChType = MkChString + fromChType (MkChString string) = string + +instance ToChType ChString Builder where + toChType = MkChString . toStrict . toLazyByteString + fromChType (MkChString string) = byteString string + +instance ToChType ChString String where + toChType = MkChString . BS8.pack + fromChType (MkChString bs)= BS8.unpack bs + +instance ToQueryPart ChString where + toQueryPart (MkChString string) = "'" <> escapeString string <> "'" + toQueryPartQuoted = toQueryPart + +escapeString :: BS.ByteString -> Builder +escapeString = byteString . BS8.concatMap espace + where + espace sym = + case sym of + '\'' -> "\\\'" + '\\' -> "\\\\" + _ -> BS8.singleton sym diff --git a/ClickHaskell/Primitive/TUInt.hs b/ClickHaskell/Primitive/TUInt.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TUInt.hs @@ -0,0 +1,131 @@ +{-# OPTIONS_GHC -Wno-orphans #-} +module ClickHaskell.Primitive.TUInt where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Data.ByteString.Builder (byteString, word8, word16LE, word32LE, word64LE) +import Data.ByteString.Char8 as BS8 (pack) +import Data.Word (Word8, Word16, Word32, Word64) + +-- External +import Data.Binary.Get (getWord8, getWord16le, getWord32le, getWord64le) +import Data.WideWord + + +-- * UInt8 + +{- | ClickHouse UInt8 column type -} +type UInt8 = Word8 + +instance IsChType UInt8 where + chTypeName = "UInt8" + defaultValueOfTypeName = 0 + +instance Serializable UInt8 where + serialize _ = word8 + deserialize _ = getWord8 + {-# INLINE deserialize #-} + +instance ToQueryPart UInt8 where + toQueryPart = byteString . BS8.pack . show + + +-- * UInt16 + + +{- | ClickHouse UInt16 column type -} +type UInt16 = Word16 + +instance IsChType UInt16 where + chTypeName = "UInt16" + defaultValueOfTypeName = 0 + +instance Serializable UInt16 where + serialize _ = word16LE + deserialize _ = getWord16le + {-# INLINE deserialize #-} + +instance ToQueryPart UInt16 where + toQueryPart = byteString . BS8.pack . show + + +-- * UInt32 + +{- | ClickHouse UInt32 column type -} +type UInt32 = Word32 + +instance IsChType UInt32 where + chTypeName = "UInt32" + defaultValueOfTypeName = 0 + +instance Serializable UInt32 where + serialize _ = word32LE + deserialize _ = getWord32le + {-# INLINE deserialize #-} + +instance ToQueryPart UInt32 where + toQueryPart = byteString . BS8.pack . show + + +-- * UInt64 + +{- | ClickHouse UInt64 column type -} +type UInt64 = Word64 + +instance IsChType UInt64 where + chTypeName = "UInt64" + defaultValueOfTypeName = 0 + +instance Serializable UInt64 where + serialize _ = word64LE + deserialize _ = getWord64le + {-# INLINE deserialize #-} + +instance ToQueryPart UInt64 where + toQueryPart = byteString . BS8.pack . show + + +-- * UInt128 + +{- | ClickHouse UInt128 column type -} +type UInt128 = Word128 + +instance IsChType UInt128 where + chTypeName = "UInt128" + defaultValueOfTypeName = 0 + +instance Serializable UInt128 where + serialize _ = (\(Word128 hi lo) -> word64LE lo <> word64LE hi) + deserialize _ = do + low <- getWord64le + high <- getWord64le + pure $ Word128 high low + {-# INLINE deserialize #-} + +instance ToQueryPart UInt128 where + toQueryPart = byteString . BS8.pack . show + + +-- * UInt256 + +{- | ClickHouse UInt128 column type -} +type UInt256 = Word256 + +instance IsChType UInt256 where + chTypeName = "UInt256" + defaultValueOfTypeName = 0 + +instance Serializable UInt256 where + serialize _ = (\(Word256 high mid1 mid0 low) -> word64LE low <> word64LE mid0 <> word64LE mid1 <> word64LE high) + deserialize _ = do + low <- getWord64le + mid0 <- getWord64le + mid1 <- getWord64le + high <- getWord64le + pure $ Word256 high mid1 mid0 low + {-# INLINE deserialize #-} + +instance ToQueryPart UInt256 where + toQueryPart = byteString . BS8.pack . show diff --git a/ClickHaskell/Primitive/TUUID.hs b/ClickHaskell/Primitive/TUUID.hs new file mode 100644 --- /dev/null +++ b/ClickHaskell/Primitive/TUUID.hs @@ -0,0 +1,47 @@ +module ClickHaskell.Primitive.TUUID where + +-- Internal +import ClickHaskell.Primitive.Serialization + +-- GHC included +import Control.DeepSeq (NFData) +import Data.Bits +import Data.ByteString.Builder (Builder, word16HexFixed, word64LE) +import Data.Word (Word64) + +-- External +import Data.Binary.Get (getWord64le) +import Data.WideWord (Word128(..)) + + +{- | ClickHouse UUID column type -} +newtype UUID = MkUUID Word128 + deriving newtype (Show, Eq, NFData, Bounded, Enum, Num) + +instance IsChType UUID where + chTypeName = "UUID" + defaultValueOfTypeName = 0 + +instance Serializable UUID where + serialize _ = (\(MkUUID (Word128 hi lo)) -> word64LE lo <> word64LE hi) + deserialize _ = do + low <- getWord64le + high <- getWord64le + pure $ MkUUID (Word128 high low) + {-# INLINE deserialize #-} + +instance ToChType UUID Word128 where + toChType = MkUUID + fromChType (MkUUID w128) = w128 + +instance ToChType UUID (Word64, Word64) where + toChType = MkUUID . uncurry (flip Word128) + fromChType (MkUUID (Word128 w64hi w64lo)) = (w64hi, w64lo) + +instance ToQueryPart UUID where + toQueryPart (MkUUID (Word128 hi lo)) = mconcat + ["'", p 3 hi, p 2 hi, "-", p 1 hi, "-", p 0 hi, "-", p 3 lo, "-", p 2 lo, p 1 lo, p 0 lo, "'"] + where + p :: Int -> Word64 -> Builder + p shiftN word = word16HexFixed $ fromIntegral (word `unsafeShiftR` (shiftN*16)) + toQueryPartQuoted = toQueryPart diff --git a/ClickHaskell/Protocol/Client.hs b/ClickHaskell/Protocol/Client.hs --- a/ClickHaskell/Protocol/Client.hs +++ b/ClickHaskell/Protocol/Client.hs @@ -3,7 +3,7 @@ -- Internal import ClickHaskell.Primitive import ClickHaskell.Protocol.Data (DataPacket) -import ClickHaskell.Protocol.Settings (DbSettings(..)) +import ClickHaskell.Protocol.Settings (DbSettings(..), DbSetting (..), appendSetting, emptySettings, SettingStringType (..), fCUSTOM, Flags, isCustom) -- GHC import Data.Int @@ -13,6 +13,9 @@ -- * Client packets +clientName :: ChString +clientName = "ClickHaskell" + data ClientPacket where Hello :: HelloPacket -> ClientPacket Query :: QueryPacket -> ClientPacket @@ -116,7 +119,7 @@ , query_stage :: QueryStage , compression :: UVarInt , query :: ChString - , parameters :: QueryParameters `SinceRevision` DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS + , params :: QueryParameters `SinceRevision` DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS } deriving (Generic, Serializable) @@ -126,11 +129,12 @@ , os_user :: ChString , query :: ChString , settings :: DbSettings + , parameters :: QueryParameters , revision :: ProtocolRevision } mkQueryPacket :: QueryPacketArgs -> ClientPacket -mkQueryPacket MkQueryPacketArgs{initial_user, os_user, hostname, query, settings, revision} = +mkQueryPacket MkQueryPacketArgs{initial_user, os_user, hostname, query, settings, revision, parameters} = Query MkQueryPacket { query_id = "" @@ -163,17 +167,36 @@ , query_stage = Complete , compression = 0 , query - , parameters = AfterRevision MkQueryParameters + , params = AfterRevision parameters , external_roles = AfterRevision 0 } -data QueryParameters = MkQueryParameters -instance Serializable QueryParameters where - serialize rev _ = - serialize @ChString rev "" - deserialize _rev = - fail "QueryParameters reading unimplemented" +-- In the ClickHouse protocol, query parameters are represented the same way as settings +newtype QueryParameters = MkQueryParameters DbSettings + deriving newtype (Serializable) + +emptyParameters :: QueryParameters +emptyParameters = MkQueryParameters emptySettings + +class IsParameterType paramType where + toParam :: Flags -> paramType -> SettingStringType + +instance ToQueryPart paramType => IsParameterType paramType + where + toParam flags param = + if isCustom flags + then (MkSettingStringType . toChType . toQueryPartQuoted) param + else (MkSettingStringType . toChType . toQueryPart) param + +addParameter :: IsParameterType param => String -> param -> QueryParameters -> QueryParameters +addParameter paramName param (MkQueryParameters settings) = + let setting = toChType paramName + flagsVal = fCUSTOM + flags = AfterRevision flagsVal + value = AfterRevision (toParam flagsVal param) + in MkQueryParameters (appendSetting MkDbSetting{..} settings) + data QueryStage = FetchColumns | WithMergeableState | Complete | WithMergeableStateAfterAggregation @@ -236,8 +259,8 @@ data Jwt = MkJwt ChString instance Serializable Jwt where - serialize rev (MkJwt str@(MkChString bs)) = - if not (BS.null bs) + serialize rev (MkJwt str@bs) = + if not (BS.null $ fromChType bs) then serialize @UInt8 rev 1 <> serialize rev str else serialize @UInt8 rev 0 deserialize rev = do diff --git a/ClickHaskell/Protocol/Data.hs b/ClickHaskell/Protocol/Data.hs --- a/ClickHaskell/Protocol/Data.hs +++ b/ClickHaskell/Protocol/Data.hs @@ -4,10 +4,25 @@ import ClickHaskell.Primitive -- GHC +import Control.Exception (Exception) +import Control.Monad (forM, when) +import Data.Binary (Get) +import Data.Bits ((.&.)) +import Data.ByteString (isPrefixOf) +import Data.ByteString.Builder (Builder, byteString, stringUtf8) +import Data.ByteString.Char8 as BS8 (pack) +import Data.Coerce (coerce) +import Data.Data (Proxy (..)) import Data.Int +import Data.Kind (Type) +import Data.List (mapAccumL) import GHC.Generics +import GHC.TypeError +import GHC.TypeLits --- * Common Data packet +------------------------------------------------------------------------------- +-- * Data packet +------------------------------------------------------------------------------- data DataPacket = MkDataPacket { table_name :: ChString @@ -36,3 +51,208 @@ , eof :: UVarInt } deriving (Generic, Serializable) + + + + +------------------------------------------------------------------------------- +-- * Columns +------------------------------------------------------------------------------- + +data Columns (columns :: [Type]) where + Empty :: Columns '[] + AddColumn + :: KnownColumn (Column name chType) + => Column name chType + -> Columns columns + -> Columns (Column name chType ': columns) + + +{- | +Column declaration + +For example: + +@ +type MyColumn = Column "myColumn" ChString +@ +-} +data Column (name :: Symbol) (chType :: Type) + +type family GetColumnName column :: Symbol where GetColumnName (Column name columnType) = name +type family GetColumnType column :: Type where GetColumnType (Column name columnType) = columnType + + +class + ( IsChType (GetColumnType column) + , KnownSymbol (GetColumnName column) + ) => + KnownColumn column where + renderColumnName :: Builder + renderColumnName = (stringUtf8 . symbolVal @(GetColumnName column)) Proxy + + renderColumnType :: Builder + renderColumnType = byteString . BS8.pack $ chTypeName @(GetColumnType column) + +data ColumnHeader = MkColumnHeader + { name :: ChString + , type_ :: ChString + , is_custom :: UInt8 `SinceRevision` DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION + } deriving (Generic, Serializable) + +mkHeader :: forall column . KnownColumn column => ColumnHeader +mkHeader = let + name = toChType $ renderColumnName @column + type_ = toChType $ chTypeName @(GetColumnType column) + is_custom = AfterRevision 0 + in MkColumnHeader{..} + +type ErrorHandler = UserError -> Get () + +validateColumnHeader :: forall column . KnownColumn column => ErrorHandler -> ProtocolRevision -> ColumnHeader -> Get () +validateColumnHeader errHandler rev MkColumnHeader{..} = do + let expectedColumnName = toChType (renderColumnName @column) + resultColumnName = name + when (resultColumnName /= expectedColumnName) $ + errHandler . UnmatchedColumn + $ "Got column \"" <> show resultColumnName <> "\" but expected \"" <> show expectedColumnName <> "\"" + + let expectedType = fallbackTypeName rev $ toChType (renderColumnType @column) + resultType = fallbackTypeName rev type_ + when (resultType /= expectedType) $ + errHandler . UnmatchedType + $ "Column " <> show resultColumnName <> " has type " <> show resultType <> ". But expected type is " <> show expectedType + +fallbackTypeName :: ProtocolRevision -> ChString -> ChString +fallbackTypeName rev typeName = toChType @ChString $ + if rev < mkRev @DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE + && isPrefixOf "DateTime(" (fromChType typeName) + then "DateTime" + else typeName + +{- | + Errors intended to be handled by developers +-} +data UserError + = UnmatchedType String + -- ^ Column type mismatch in data packet + | UnmatchedColumn String + -- ^ Column name mismatch in data packet + | UnmatchedColumnsCount String + -- ^ Occurs when actual columns count less or more than expected + deriving (Show, Exception) + + + + +------------------------------------------------------------------------------- +-- * Columns serialization +------------------------------------------------------------------------------- + +type ErrHandler = UserError -> Get () + +ignoreErr :: ErrHandler +ignoreErr _ = pure () + +class SerializableColumn col where + {-# INLINE deserializeColumn #-} + deserializeColumn :: KnownColumn col => ErrHandler -> ProtocolRevision -> UVarInt -> (GetColumnType col -> res) -> Get [res] + deserializeColumn errHandler rev size f = do + validateColumnHeader @col errHandler rev =<< deserialize @ColumnHeader rev + deserializeColumnI @col rev size f + + {-# INLINE serializeColumn #-} + serializeColumn :: KnownColumn col => ProtocolRevision -> (a -> GetColumnType col) -> [a] -> Builder + serializeColumn rev f values = + serialize rev (mkHeader @col) <> + serializeColumnI @col rev (f) values + + deserializeColumnI :: ProtocolRevision -> UVarInt -> (GetColumnType col -> a) -> Get [a] + serializeColumnI :: ProtocolRevision -> (a -> GetColumnType col) -> [a] -> Builder + +instance (IsChType chType, KnownSymbol name) => KnownColumn (Column name chType) + +instance + ( Serializable chType + , IsChType chType + ) => + SerializableColumn (Column name chType) where + {-# INLINE deserializeColumnI #-} + deserializeColumnI rev rows f = map f <$> replicateGet rev rows + + {-# INLINE serializeColumnI #-} + serializeColumnI rev f column = foldMap (serialize @chType rev . f) column + + +instance {-# OVERLAPPING #-} + ( KnownColumn (Column name (Nullable chType)) + , Serializable chType + , IsChType chType + ) => + SerializableColumn (Column name (Nullable chType)) where + {-# INLINE deserializeColumnI #-} + deserializeColumnI rev rows f = do + nulls <- replicateGet @UInt8 rev rows + forM nulls (\nulFlag -> case nulFlag of + 0 -> f . Just <$> deserialize @chType rev + _ -> (f Nothing <$ deserialize @chType rev) + ) + + {-# INLINE serializeColumnI #-} + serializeColumnI rev f column + = foldMap (serialize @UInt8 rev . maybe 1 (const 0) . f) column + <> foldMap (serialize @chType rev . maybe defaultValueOfTypeName id . f) column + +instance {-# OVERLAPPING #-} + ( KnownColumn (Column name (LowCardinality chType)) + , Serializable chType + , IsLowCardinalitySupported chType + , TypeError ('Text "LowCardinality deserialization still unsupported") + ) => + SerializableColumn (Column name (LowCardinality chType)) where + {-# INLINE deserializeColumnI #-} + deserializeColumnI rev rows f = do + _serializationType <- (.&. 0xf) <$> deserialize @UInt64 rev + _index_size <- deserialize @Int64 rev + -- error $ "Trace | " <> show _serializationType <> " : " <> show _index_size + map f . coerce + <$> replicateGet @chType rev rows + + {-# INLINE serializeColumnI #-} + serializeColumnI _rev column = undefined column + +instance {-# OVERLAPPING #-} + ( KnownColumn (Column name (Array chType)) + , Serializable chType + ) + => SerializableColumn (Column name (Array chType)) where + {-# INLINE deserializeColumnI #-} + deserializeColumnI rev rows f = do + offsets <- replicateGet @UInt64 rev rows + let lengths = zipWith (-) offsets (0 : (init offsets)) + forM lengths (fmap (f . MkChArray) . replicateGet @chType rev . fromIntegral) + + {-# INLINE serializeColumnI #-} + serializeColumnI rev f column + = foldMap (serialize @UInt64 rev) offsets + <> foldMap (foldMap (serialize @chType rev) . f) column + where + offsets = + snd $ + mapAccumL + (\offset xs -> + let nextOffset = offset + fromIntegral (length xs) + in (nextOffset, nextOffset) + ) + 0 + (map f column) + +instance {-# OVERLAPPING #-} + ( KnownSymbol name + , IsChType chType + , TypeError ('Text "Nested Arrays types (column \"" :<>: 'Text name :<>: 'Text "\") are unsupported") + ) + => SerializableColumn (Column name (Array (Array chType))) + where + deserializeColumnI = error "Impossible" + serializeColumnI = error "Impossible" diff --git a/ClickHaskell/Protocol/Server.hs b/ClickHaskell/Protocol/Server.hs --- a/ClickHaskell/Protocol/Server.hs +++ b/ClickHaskell/Protocol/Server.hs @@ -2,13 +2,12 @@ -- Internal import ClickHaskell.Primitive -import ClickHaskell.Protocol.Data (DataPacket(..)) +import ClickHaskell.Protocol.Data (DataPacket(..), Column, SerializableColumn (..), ignoreErr) import ClickHaskell.Protocol.Settings (DbSettings) -- GHC import Data.Int import GHC.Generics -import ClickHaskell.Columns (Column, deserializeColumn, ColumnHeader, serializeColumn, mkHeader) import Control.Monad (when) -- * Server packets @@ -157,25 +156,24 @@ , value :: [UInt64] } deriving (Generic) --- ToDo: Simplify instance Serializable ProfileEventsPacket where serialize rev MkProfileEventsPacket{..} = serialize rev dataPacket - <> serialize rev (mkHeader @(Column "host_name" ChString)) <> serializeColumn @(Column "host_name" ChString) rev id host_name - <> serialize rev (mkHeader @(Column "current_time" (DateTime ""))) <> serializeColumn @(Column "current_time" (DateTime "")) rev id current_time - <> serialize rev (mkHeader @(Column "thread_id" UInt64)) <> serializeColumn @(Column "thread_id" UInt64) rev id thread_id - <> serialize rev (mkHeader @(Column "type" Int8)) <> serializeColumn @(Column "type" Int8) rev id type_ - <> serialize rev (mkHeader @(Column "name" ChString)) <> serializeColumn @(Column "name" ChString) rev id name - <> serialize rev (mkHeader @(Column "value" UInt64)) <> serializeColumn @(Column "value" UInt64) rev id value + <> serializeColumn @(Column "host_name" ChString) rev id host_name + <> serializeColumn @(Column "current_time" (DateTime "")) rev id current_time + <> serializeColumn @(Column "thread_id" UInt64) rev id thread_id + <> serializeColumn @(Column "type" Int8) rev id type_ + <> serializeColumn @(Column "name" ChString) rev id name + <> serializeColumn @(Column "value" UInt64) rev id value deserialize rev = do dataPacket@MkDataPacket{rows_count, columns_count} <- deserialize rev validateColumnsCount columns_count - !host_name <- deserialize @ColumnHeader rev *> deserializeColumn @(Column "host_name" ChString) rev rows_count id - !current_time <- deserialize @ColumnHeader rev *> deserializeColumn @(Column "current_time" (DateTime "")) rev rows_count id - !thread_id <- deserialize @ColumnHeader rev *> deserializeColumn @(Column "thread_id" UInt64) rev rows_count id - !type_ <- deserialize @ColumnHeader rev *> deserializeColumn @(Column "type" Int8) rev rows_count id - !name <- deserialize @ColumnHeader rev *> deserializeColumn @(Column "name" ChString) rev rows_count id - !value <- deserialize @ColumnHeader rev *> deserializeColumn @(Column "value" UInt64) rev rows_count id + !host_name <- deserializeColumn @(Column "host_name" ChString) ignoreErr rev rows_count id + !current_time <- deserializeColumn @(Column "current_time" (DateTime "")) ignoreErr rev rows_count id + !thread_id <- deserializeColumn @(Column "thread_id" UInt64) ignoreErr rev rows_count id + !type_ <- deserializeColumn @(Column "type" Int8) ignoreErr rev rows_count id + !name <- deserializeColumn @(Column "name" ChString) ignoreErr rev rows_count id + !value <- deserializeColumn @(Column "value" UInt64) ignoreErr rev rows_count id pure $ MkProfileEventsPacket{..} where validateColumnsCount count = when (count /= 6) . fail $ diff --git a/ClickHaskell/Protocol/Settings.hs b/ClickHaskell/Protocol/Settings.hs --- a/ClickHaskell/Protocol/Settings.hs +++ b/ClickHaskell/Protocol/Settings.hs @@ -13,6 +13,7 @@ import Data.Bool (bool) import Data.ByteString as BS (null) import Data.ByteString.Builder (Builder) +import Data.Int (Int64) import Data.Kind (Type) import Data.Proxy (Proxy (..)) import GHC.TypeLits @@ -22,17 +23,23 @@ data DbSettings = MkDbSettings [DbSetting] +emptySettings :: DbSettings +emptySettings = MkDbSettings [] + +appendSetting :: DbSetting -> DbSettings -> DbSettings +appendSetting dbSetting (MkDbSettings xs) = MkDbSettings (dbSetting : xs) + addSetting :: forall name settType . KnownSetting name settType => settType -> DbSettings -> DbSettings -addSetting val (MkDbSettings xs) = +addSetting val settings = let setting = toChType (symbolVal @name Proxy) flags = AfterRevision fIMPORTANT - value = toSettingType val - in MkDbSettings (MkDbSetting{..} : xs) + value = BeforeRevision (toSettingType val) + in appendSetting MkDbSetting{..} settings data DbSetting = MkDbSetting { setting :: ChString @@ -44,30 +51,42 @@ deserialize rev = do setting <- deserialize @ChString rev flags <- deserialize @(Flags `SinceRevision` DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) rev - case lookup setting settingsMap of - Nothing -> fail ("Unsupported setting " <> show setting) - Just MkSettingSerializer{deserializer} -> do - value <- deserializer rev - pure $ MkDbSetting{..} + value <- + if rev < mkRev @DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS + then BeforeRevision <$> (deserializeBinary setting) + else AfterRevision <$> (deserializeString) + pure $ MkDbSetting{..} + where + deserializeBinary setting = do + case lookup setting settingsMap of + Nothing -> fail ("Unsupported setting " <> show setting) + Just MkSettingSerializer{deserializer} -> deserializer rev + deserializeString = deserialize @SettingStringType rev + serialize rev MkDbSetting{setting, flags, value} = serialize rev setting <> serialize rev flags - <> case lookup setting settingsMap of - Nothing -> error "Impossible happened. Unknown setting was added to query packet" - Just MkSettingSerializer{serializer} -> serializer rev value + <> serializedValue + where + serializedValue :: Builder + serializedValue = case value of + (BeforeRevision sett) -> + case lookup setting settingsMap of + Nothing -> error "Impossible happened. Unknown setting was added to query packet" + Just MkSettingSerializer{serializer} -> serializer rev sett + (AfterRevision sett) -> serialize rev sett instance Serializable DbSettings where serialize rev (MkDbSettings setts) = foldMap (serialize @DbSetting rev) setts <> serialize @ChString rev "" deserialize rev = do - (MkChString setting) <- lookAhead (deserialize @ChString rev) - if BS.null setting + setting <- lookAhead (deserialize @ChString rev) + if BS.null (fromChType setting) then deserialize @ChString rev *> pure (MkDbSettings []) else do sett <- deserialize @DbSetting rev - (\(MkDbSettings setts) -> MkDbSettings (sett : setts)) - <$> deserialize @DbSettings rev + appendSetting sett <$> deserialize @DbSettings rev -- ** Flags @@ -107,30 +126,64 @@ => IsSettingType settType where - toSettingType :: settType -> SettingType - fromSettingType :: SettingType -> settType + toSettingType :: settType -> SettingBinaryType + fromSettingType :: SettingBinaryType -> settType - serializeSettingText :: ProtocolRevision -> SettingType -> Builder - serializeSettingText rev = serialize @ChString rev . toChType . toQueryPart @settType . fromSettingType + toSettingString :: SettingBinaryType -> SettingStringType + toSettingString = MkSettingStringType . toChType . toQueryPart @settType . fromSettingType - serializeSettingBinary :: ProtocolRevision -> SettingType -> Builder + toSettingStringQuoted :: SettingBinaryType -> SettingStringType + toSettingStringQuoted = MkSettingStringType . toChType . toQueryPartQuoted @settType . fromSettingType -data SettingType where - SettingUInt64 :: UInt64 -> SettingType - SettingString :: ChString -> SettingType - SettingBool :: Bool -> SettingType + serializeSettingBinary :: ProtocolRevision -> SettingBinaryType -> Builder +type SettingType = + Revisioned + DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS + SettingBinaryType + SettingStringType + +newtype SettingStringType = MkSettingStringType ChString + deriving newtype (Serializable) + + +data SettingBinaryType where + SettingUInt64 :: UInt64 -> SettingBinaryType + SettingInt64 :: Int64 -> SettingBinaryType + SettingFloat :: Float -> SettingBinaryType + SettingDouble :: Double -> SettingBinaryType + SettingString :: ChString -> SettingBinaryType + SettingBool :: Bool -> SettingBinaryType + instance IsSettingType ChString where toSettingType str = SettingString str fromSettingType (SettingString str) = str fromSettingType _ = error "Impossible" serializeSettingBinary rev = serialize @ChString rev . fromSettingType +instance IsSettingType Int64 where + toSettingType int64 = SettingInt64 int64 + fromSettingType (SettingInt64 int64) = int64 + fromSettingType _ = error "Impossible" + serializeSettingBinary rev = serialize @VarInt rev . fromIntegral . fromSettingType @Int64 + +instance IsSettingType Float where + toSettingType float = SettingFloat float + fromSettingType (SettingFloat float) = float + fromSettingType _ = error "Impossible" + serializeSettingBinary rev = serialize @SettingStringType rev . toSettingString @Float + +instance IsSettingType Double where + toSettingType float = SettingDouble float + fromSettingType (SettingDouble float) = float + fromSettingType _ = error "Impossible" + serializeSettingBinary rev = serialize @SettingStringType rev . toSettingString @Double + instance IsSettingType UInt64 where toSettingType uint64 = SettingUInt64 uint64 fromSettingType (SettingUInt64 uint64) = uint64 fromSettingType _ = error "Impossible" - serializeSettingBinary rev = serialize @UVarInt rev . fromIntegral @UInt64 . fromSettingType + serializeSettingBinary rev = serialize @UVarInt rev . fromIntegral . fromSettingType @UInt64 instance IsSettingType Bool where toSettingType boolean = SettingBool boolean @@ -141,8 +194,8 @@ data SettingSerializer = MkSettingSerializer - { deserializer :: ProtocolRevision -> Get SettingType - , serializer :: ProtocolRevision -> SettingType -> Builder + { deserializer :: ProtocolRevision -> Get SettingBinaryType + , serializer :: ProtocolRevision -> SettingBinaryType -> Builder } class @@ -151,24 +204,26 @@ ) => KnownSetting name settType | name -> settType - where - mkSettingSerializer :: (ChString, SettingSerializer) - mkSettingSerializer = - let name = toChType (symbolVal @name Proxy) - deserializer = \rev -> - if rev >= mkRev @DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS - then fail "Deserialization of Settings serializaed as strings is unsuported" - else toSettingType <$> deserialize @settType rev - serializer = \rev -> - if rev >= mkRev @DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS - then serializeSettingText @settType rev - else serializeSettingBinary @settType rev - in (name, MkSettingSerializer {deserializer, serializer}) +{-# NOINLINE mkSettingSerializer #-} +mkSettingSerializer :: forall name settType . KnownSetting name settType => (ChString, SettingSerializer) +mkSettingSerializer = + let name = toChType (symbolVal @name Proxy) + deserializer = \rev -> + if rev >= mkRev @DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS + then fail "Deserialization of Settings serializaed as strings is unsuported" + else toSettingType <$> deserialize @settType rev + serializer = \rev -> + if rev >= mkRev @DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS + then serialize rev . toSettingString @settType + else serializeSettingBinary @settType rev + in (name, MkSettingSerializer {deserializer, serializer}) + data Setting (a :: Symbol) (settType :: Type) instance KnownSetting "min_compress_block_size" UInt64 instance KnownSetting "max_compress_block_size" UInt64 +instance KnownSetting "max_insert_block_size_bytes" UInt64 instance KnownSetting "min_insert_block_size_rows" UInt64 instance KnownSetting "min_insert_block_size_bytes" UInt64 instance KnownSetting "min_insert_block_size_rows_for_materialized_views" UInt64 @@ -256,6 +311,7 @@ instance KnownSetting "azure_request_timeout_ms" UInt64 instance KnownSetting "azure_connect_timeout_ms" UInt64 instance KnownSetting "s3_validate_request_settings" Bool +instance KnownSetting "compatibility_s3_presigned_url_query_in_path" Bool instance KnownSetting "s3_disable_checksum" Bool instance KnownSetting "s3_request_timeout_ms" UInt64 instance KnownSetting "s3_connect_timeout_ms" UInt64 @@ -263,6 +319,7 @@ instance KnownSetting "s3queue_default_zookeeper_path" ChString instance KnownSetting "s3queue_migrate_old_metadata_to_buckets" Bool instance KnownSetting "s3queue_enable_logging_to_s3queue_log" Bool +instance KnownSetting "s3queue_keeper_fault_injection_probability" Float instance KnownSetting "hdfs_replication" UInt64 instance KnownSetting "hdfs_truncate_on_insert" Bool instance KnownSetting "hdfs_create_new_file_on_insert" Bool @@ -287,8 +344,10 @@ instance KnownSetting "enable_multiple_prewhere_read_steps" Bool instance KnownSetting "move_primary_key_columns_to_end_of_prewhere" Bool instance KnownSetting "allow_reorder_prewhere_conditions" Bool +instance KnownSetting "replication_wait_for_inactive_replica_timeout" Int64 instance KnownSetting "alter_move_to_space_execute_async" Bool instance KnownSetting "load_balancing_first_offset" UInt64 +instance KnownSetting "totals_auto_threshold" Float instance KnownSetting "allow_suspicious_low_cardinality_types" Bool instance KnownSetting "allow_suspicious_fixed_string_types" Bool instance KnownSetting "allow_suspicious_indices" Bool @@ -297,8 +356,7 @@ instance KnownSetting "allow_suspicious_primary_key" Bool instance KnownSetting "allow_suspicious_types_in_group_by" Bool instance KnownSetting "allow_suspicious_types_in_order_by" Bool -instance KnownSetting "allow_not_comparable_types_in_order_by" Bool -instance KnownSetting "allow_not_comparable_types_in_comparison_functions" Bool +instance KnownSetting "use_variant_default_implementation_for_comparisons" Bool instance KnownSetting "compile_expressions" Bool instance KnownSetting "min_count_to_compile_expression" UInt64 instance KnownSetting "compile_aggregate_expressions" Bool @@ -311,6 +369,7 @@ instance KnownSetting "aggregation_memory_efficient_merge_threads" UInt64 instance KnownSetting "enable_memory_bound_merging_of_aggregation_results" Bool instance KnownSetting "enable_positional_arguments" Bool +instance KnownSetting "enable_positional_arguments_for_projections" Bool instance KnownSetting "enable_extended_results_for_datetime_functions" Bool instance KnownSetting "allow_nonconst_timezone_arguments" Bool instance KnownSetting "use_legacy_to_time" Bool @@ -328,10 +387,6 @@ instance KnownSetting "force_optimize_skip_unused_shards" UInt64 instance KnownSetting "optimize_skip_unused_shards_nesting" UInt64 instance KnownSetting "force_optimize_skip_unused_shards_nesting" UInt64 -instance KnownSetting "input_format_parallel_parsing" Bool -instance KnownSetting "output_format_parallel_formatting" Bool -instance KnownSetting "output_format_compression_level" UInt64 -instance KnownSetting "output_format_compression_zstd_window_log" UInt64 instance KnownSetting "allow_special_serialization_kinds_in_output_formats" Bool instance KnownSetting "enable_parsing_to_custom_serialization" Bool instance KnownSetting "merge_tree_use_v1_object_and_dynamic_serialization" Bool @@ -347,6 +402,8 @@ instance KnownSetting "do_not_merge_across_partitions_select_final" Bool instance KnownSetting "split_parts_ranges_into_intersecting_and_non_intersecting_final" Bool instance KnownSetting "split_intersecting_parts_ranges_into_layers_final" Bool +instance KnownSetting "apply_row_policy_after_final" Bool +instance KnownSetting "apply_prewhere_after_final" Bool instance KnownSetting "mysql_max_rows_to_insert" UInt64 instance KnownSetting "mysql_map_string_to_text_in_show_columns" Bool instance KnownSetting "mysql_map_fixed_string_to_text_in_show_columns" Bool @@ -356,11 +413,16 @@ instance KnownSetting "min_bytes_to_use_mmap_io" UInt64 instance KnownSetting "checksum_on_read" Bool instance KnownSetting "force_index_by_date" Bool +instance KnownSetting "use_primary_key" Bool instance KnownSetting "force_primary_key" Bool instance KnownSetting "use_skip_indexes" Bool instance KnownSetting "use_skip_indexes_if_final" Bool instance KnownSetting "use_skip_indexes_if_final_exact_mode" Bool instance KnownSetting "use_skip_indexes_on_data_read" Bool +instance KnownSetting "use_skip_indexes_for_disjunctions" Bool +instance KnownSetting "use_skip_indexes_for_top_k" Bool +instance KnownSetting "use_top_k_dynamic_filtering" Bool +instance KnownSetting "query_plan_max_limit_for_top_k_optimization" UInt64 instance KnownSetting "materialize_skip_indexes_on_insert" Bool instance KnownSetting "exclude_materialize_skip_indexes_on_insert" ChString instance KnownSetting "text_index_use_bloom_filter" Bool @@ -369,11 +431,16 @@ instance KnownSetting "ignore_data_skipping_indices" ChString instance KnownSetting "force_data_skipping_indices" ChString instance KnownSetting "secondary_indices_enable_bulk_filtering" Bool +instance KnownSetting "max_streams_to_max_threads_ratio" Float +instance KnownSetting "max_streams_multiplier_for_merge_tables" Float instance KnownSetting "network_compression_method" ChString +instance KnownSetting "network_zstd_compression_level" Int64 +instance KnownSetting "zstd_window_log_max" Int64 instance KnownSetting "priority" UInt64 instance KnownSetting "log_queries" Bool instance KnownSetting "log_formatted_queries" Bool instance KnownSetting "log_queries_cut_to_length" UInt64 +instance KnownSetting "log_queries_probability" Float instance KnownSetting "log_processors_profiles" Bool instance KnownSetting "max_concurrent_queries_for_all_users" UInt64 instance KnownSetting "max_concurrent_queries_for_user" UInt64 @@ -386,12 +453,16 @@ instance KnownSetting "read_backoff_max_throughput" UInt64 instance KnownSetting "read_backoff_min_events" UInt64 instance KnownSetting "read_backoff_min_concurrency" UInt64 +instance KnownSetting "memory_tracker_fault_probability" Float +instance KnownSetting "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability" Float instance KnownSetting "enable_http_compression" Bool +instance KnownSetting "http_zlib_compression_level" Int64 instance KnownSetting "http_native_compression_disable_checksumming_on_decompress" Bool instance KnownSetting "count_distinct_implementation" ChString instance KnownSetting "add_http_cors_header" Bool instance KnownSetting "max_http_get_redirects" UInt64 instance KnownSetting "use_client_time_zone" Bool +instance KnownSetting "send_profile_events" Bool instance KnownSetting "send_progress_in_http_headers" Bool instance KnownSetting "http_headers_progress_interval_ms" UInt64 instance KnownSetting "http_wait_end_of_query" Bool @@ -414,7 +485,9 @@ instance KnownSetting "parts_to_throw_insert" UInt64 instance KnownSetting "number_of_mutations_to_delay" UInt64 instance KnownSetting "number_of_mutations_to_throw" UInt64 +instance KnownSetting "distributed_ddl_task_timeout" Int64 instance KnownSetting "min_free_disk_bytes_to_perform_insert" UInt64 +instance KnownSetting "min_free_disk_ratio_to_perform_insert" Float instance KnownSetting "final" Bool instance KnownSetting "partial_result_on_first_cancel" Bool instance KnownSetting "ignore_on_cluster_for_replicated_udf_queries" Bool @@ -437,12 +510,11 @@ instance KnownSetting "empty_result_for_aggregation_by_constant_keys_on_empty_set" Bool instance KnownSetting "allow_distributed_ddl" Bool instance KnownSetting "allow_suspicious_codecs" Bool -instance KnownSetting "enable_zstd_qat_codec" Bool -instance KnownSetting "enable_deflate_qpl_codec" Bool instance KnownSetting "query_profiler_real_time_period_ns" UInt64 instance KnownSetting "query_profiler_cpu_time_period_ns" UInt64 instance KnownSetting "metrics_perf_events_enabled" Bool instance KnownSetting "metrics_perf_events_list" ChString +instance KnownSetting "opentelemetry_start_trace_probability" Float instance KnownSetting "opentelemetry_trace_processors" Bool instance KnownSetting "opentelemetry_trace_cpu_scheduling" Bool instance KnownSetting "prefer_column_name_to_alias" Bool @@ -455,11 +527,14 @@ instance KnownSetting "max_bytes_to_read_leaf" UInt64 instance KnownSetting "max_rows_to_group_by" UInt64 instance KnownSetting "max_bytes_before_external_group_by" UInt64 +instance KnownSetting "max_bytes_ratio_before_external_group_by" Double instance KnownSetting "max_rows_to_sort" UInt64 instance KnownSetting "max_bytes_to_sort" UInt64 instance KnownSetting "prefer_external_sort_block_bytes" UInt64 instance KnownSetting "max_bytes_before_external_sort" UInt64 +instance KnownSetting "max_bytes_ratio_before_external_sort" Double instance KnownSetting "max_bytes_before_remerge_sort" UInt64 +instance KnownSetting "remerge_sort_lowered_memory_bytes_ratio" Float instance KnownSetting "max_result_rows" UInt64 instance KnownSetting "max_result_bytes" UInt64 instance KnownSetting "min_execution_speed" UInt64 @@ -478,9 +553,12 @@ instance KnownSetting "readonly" UInt64 instance KnownSetting "max_rows_in_set" UInt64 instance KnownSetting "max_bytes_in_set" UInt64 +instance KnownSetting "exact_rows_before_limit" Bool +instance KnownSetting "rows_before_aggregation" Bool instance KnownSetting "max_rows_in_join" UInt64 instance KnownSetting "max_bytes_in_join" UInt64 instance KnownSetting "join_any_take_last_row" Bool +instance KnownSetting "cross_to_inner_join_rewrite" UInt64 instance KnownSetting "cross_join_min_rows_to_compress" UInt64 instance KnownSetting "cross_join_min_bytes_to_compress" UInt64 instance KnownSetting "default_max_bytes_in_join" UInt64 @@ -490,7 +568,6 @@ instance KnownSetting "max_rows_in_set_to_optimize_join" UInt64 instance KnownSetting "compatibility_ignore_collation_in_create_table" Bool instance KnownSetting "temporary_files_codec" ChString -instance KnownSetting "temporary_files_buffer_size" UInt64 instance KnownSetting "max_rows_to_transfer" UInt64 instance KnownSetting "max_bytes_to_transfer" UInt64 instance KnownSetting "max_rows_in_distinct" UInt64 @@ -500,10 +577,13 @@ instance KnownSetting "max_memory_usage_for_user" UInt64 instance KnownSetting "memory_overcommit_ratio_denominator_for_user" UInt64 instance KnownSetting "max_untracked_memory" UInt64 +instance KnownSetting "max_reverse_dictionary_lookup_cache_size_bytes" UInt64 instance KnownSetting "memory_profiler_step" UInt64 +instance KnownSetting "memory_profiler_sample_probability" Float instance KnownSetting "memory_profiler_sample_min_allocation_size" UInt64 instance KnownSetting "memory_profiler_sample_max_allocation_size" UInt64 instance KnownSetting "trace_profile_events" Bool +instance KnownSetting "trace_profile_events_list" ChString instance KnownSetting "memory_usage_overcommit_max_wait_microseconds" UInt64 instance KnownSetting "max_network_bandwidth" UInt64 instance KnownSetting "max_network_bytes" UInt64 @@ -521,10 +601,12 @@ instance KnownSetting "backup_restore_keeper_value_max_size" UInt64 instance KnownSetting "backup_restore_batch_size_for_keeper_multi" UInt64 instance KnownSetting "backup_restore_batch_size_for_keeper_multiread" UInt64 +instance KnownSetting "backup_restore_keeper_fault_injection_probability" Float instance KnownSetting "backup_restore_keeper_fault_injection_seed" UInt64 instance KnownSetting "backup_restore_s3_retry_attempts" UInt64 instance KnownSetting "backup_restore_s3_retry_initial_backoff_ms" UInt64 instance KnownSetting "backup_restore_s3_retry_max_backoff_ms" UInt64 +instance KnownSetting "backup_restore_s3_retry_jitter_factor" Float instance KnownSetting "max_backup_bandwidth" UInt64 instance KnownSetting "restore_replicated_merge_tree_to_shared_merge_tree" Bool instance KnownSetting "log_profile_events" Bool @@ -532,6 +614,11 @@ instance KnownSetting "log_query_threads" Bool instance KnownSetting "log_query_views" Bool instance KnownSetting "log_comment" ChString +instance KnownSetting "query_metric_log_interval" Int64 +instance KnownSetting "regexp_dict_allow_hyperscan" Bool +instance KnownSetting "regexp_dict_flag_case_insensitive" Bool +instance KnownSetting "regexp_dict_flag_dotall" Bool +instance KnownSetting "dictionary_use_async_executor" Bool instance KnownSetting "send_logs_source_regexp" ChString instance KnownSetting "enable_optimize_predicate_expression" Bool instance KnownSetting "enable_optimize_predicate_expression_to_final_subquery" Bool @@ -579,6 +666,7 @@ instance KnownSetting "h3togeo_lon_lat_result_order" Bool instance KnownSetting "max_partitions_per_insert_block" UInt64 instance KnownSetting "throw_on_max_partitions_per_insert_block" Bool +instance KnownSetting "max_partitions_to_read" Int64 instance KnownSetting "check_query_single_value_result" Bool instance KnownSetting "allow_drop_detached" Bool instance KnownSetting "max_parts_to_move" UInt64 @@ -589,6 +677,7 @@ instance KnownSetting "postgresql_connection_pool_wait_timeout" UInt64 instance KnownSetting "postgresql_connection_pool_retries" UInt64 instance KnownSetting "postgresql_connection_pool_auto_close_connection" Bool +instance KnownSetting "postgresql_fault_injection_probability" Float instance KnownSetting "glob_expansion_max_elements" UInt64 instance KnownSetting "odbc_bridge_connection_pool_size" UInt64 instance KnownSetting "odbc_bridge_use_connection_pooling" Bool @@ -625,7 +714,6 @@ instance KnownSetting "normalize_function_names" Bool instance KnownSetting "enable_early_constant_folding" Bool instance KnownSetting "deduplicate_blocks_in_dependent_materialized_views" Bool -instance KnownSetting "throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert" Bool instance KnownSetting "materialized_views_ignore_errors" Bool instance KnownSetting "ignore_materialized_views_with_dropped_target_table" Bool instance KnownSetting "allow_materialized_view_with_bad_select" Bool @@ -667,7 +755,6 @@ instance KnownSetting "async_socket_for_remote" Bool instance KnownSetting "async_query_sending_for_remote" Bool instance KnownSetting "insert_null_as_default" Bool -instance KnownSetting "describe_extend_object_types" Bool instance KnownSetting "describe_include_subcolumns" Bool instance KnownSetting "describe_include_virtual_columns" Bool instance KnownSetting "describe_compact_output" Bool @@ -676,6 +763,7 @@ instance KnownSetting "mutations_execute_nondeterministic_on_initiator" Bool instance KnownSetting "mutations_execute_subqueries_on_initiator" Bool instance KnownSetting "mutations_max_literal_size_to_replace" UInt64 +instance KnownSetting "create_replicated_merge_tree_fault_injection_probability" Float instance KnownSetting "delta_lake_log_metadata" Bool instance KnownSetting "iceberg_delete_data_on_drop" Bool instance KnownSetting "use_iceberg_metadata_files_cache" Bool @@ -705,6 +793,7 @@ instance KnownSetting "collect_hash_table_stats_during_aggregation" Bool instance KnownSetting "max_size_to_preallocate_for_aggregation" UInt64 instance KnownSetting "collect_hash_table_stats_during_joins" Bool +instance KnownSetting "use_hash_table_stats_for_join_reordering" Bool instance KnownSetting "max_size_to_preallocate_for_joins" UInt64 instance KnownSetting "kafka_disable_num_consumers_limit" Bool instance KnownSetting "allow_experimental_kafka_offsets_storage_in_keeper" Bool @@ -712,6 +801,7 @@ instance KnownSetting "allow_aggregate_partitions_independently" Bool instance KnownSetting "force_aggregate_partitions_independently" Bool instance KnownSetting "max_number_of_partitions_for_independent_aggregation" UInt64 +instance KnownSetting "min_hit_rate_to_use_consecutive_keys_optimization" Float instance KnownSetting "engine_file_empty_if_not_exists" Bool instance KnownSetting "engine_file_truncate_on_insert" Bool instance KnownSetting "engine_file_allow_create_multiple_files" Bool @@ -726,6 +816,7 @@ instance KnownSetting "database_replicated_allow_replicated_engine_arguments" UInt64 instance KnownSetting "database_replicated_allow_explicit_uuid" UInt64 instance KnownSetting "database_replicated_allow_heavy_create" Bool +instance KnownSetting "database_shared_drop_table_delay_seconds" UInt64 instance KnownSetting "cloud_mode" Bool instance KnownSetting "cloud_mode_engine" UInt64 instance KnownSetting "cloud_mode_database_engine" UInt64 @@ -758,6 +849,7 @@ instance KnownSetting "query_plan_reuse_storage_ordering_for_window_functions" Bool instance KnownSetting "query_plan_lift_up_union" Bool instance KnownSetting "query_plan_read_in_order" Bool +instance KnownSetting "query_plan_read_in_order_through_join" Bool instance KnownSetting "query_plan_aggregation_in_order" Bool instance KnownSetting "query_plan_remove_redundant_sorting" Bool instance KnownSetting "query_plan_remove_redundant_distinct" Bool @@ -768,6 +860,7 @@ instance KnownSetting "enable_lazy_columns_replication" Bool instance KnownSetting "serialize_query_plan" Bool instance KnownSetting "correlated_subqueries_substitute_equivalent_expressions" Bool +instance KnownSetting "correlated_subqueries_use_in_memory_buffer" Bool instance KnownSetting "optimize_qbit_distance_function_reads" Bool instance KnownSetting "regexp_max_matches_per_row" UInt64 instance KnownSetting "limit" UInt64 @@ -779,6 +872,7 @@ instance KnownSetting "remote_filesystem_read_method" ChString instance KnownSetting "local_filesystem_read_prefetch" Bool instance KnownSetting "remote_filesystem_read_prefetch" Bool +instance KnownSetting "read_priority" Int64 instance KnownSetting "merge_tree_min_rows_for_concurrent_read_for_remote_filesystem" UInt64 instance KnownSetting "merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem" UInt64 instance KnownSetting "remote_read_min_bytes_for_seek" UInt64 @@ -790,9 +884,12 @@ instance KnownSetting "async_insert_max_data_size" UInt64 instance KnownSetting "async_insert_max_query_number" UInt64 instance KnownSetting "async_insert_use_adaptive_busy_timeout" Bool +instance KnownSetting "async_insert_busy_timeout_increase_rate" Double +instance KnownSetting "async_insert_busy_timeout_decrease_rate" Double instance KnownSetting "remote_fs_read_max_backoff_ms" UInt64 instance KnownSetting "remote_fs_read_backoff_max_tries" UInt64 instance KnownSetting "cluster_function_process_archive_on_multiple_nodes" Bool +instance KnownSetting "max_streams_for_files_processing_in_cluster_functions" UInt64 instance KnownSetting "enable_filesystem_cache" Bool instance KnownSetting "filesystem_cache_name" ChString instance KnownSetting "enable_filesystem_cache_on_write_operations" Bool @@ -869,11 +966,16 @@ instance KnownSetting "distributed_cache_receive_timeout_ms" UInt64 instance KnownSetting "distributed_cache_send_timeout_ms" UInt64 instance KnownSetting "distributed_cache_tcp_keep_alive_timeout_ms" UInt64 +instance KnownSetting "distributed_cache_use_clients_cache_for_write" Bool +instance KnownSetting "distributed_cache_use_clients_cache_for_read" Bool +instance KnownSetting "distributed_cache_file_cache_name" ChString +instance KnownSetting "filesystem_cache_allow_background_download" Bool instance KnownSetting "filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage" Bool instance KnownSetting "filesystem_cache_enable_background_download_during_fetch" Bool instance KnownSetting "parallelize_output_from_storages" Bool instance KnownSetting "insert_deduplication_token" ChString instance KnownSetting "count_distinct_optimization" Bool +instance KnownSetting "optimize_inverse_dictionary_lookup" Bool instance KnownSetting "throw_if_no_data_to_insert" Bool instance KnownSetting "compatibility_ignore_auto_increment_in_create_table" Bool instance KnownSetting "multiple_joins_try_to_keep_original_names" Bool @@ -884,6 +986,7 @@ instance KnownSetting "insert_keeper_max_retries" UInt64 instance KnownSetting "insert_keeper_retry_initial_backoff_ms" UInt64 instance KnownSetting "insert_keeper_retry_max_backoff_ms" UInt64 +instance KnownSetting "insert_keeper_fault_injection_probability" Float instance KnownSetting "insert_keeper_fault_injection_seed" UInt64 instance KnownSetting "force_aggregation_in_order" Bool instance KnownSetting "http_max_request_param_data_size" UInt64 @@ -893,6 +996,7 @@ instance KnownSetting "optimize_uniq_to_count" Bool instance KnownSetting "use_variant_as_common_type" Bool instance KnownSetting "enable_order_by_all" Bool +instance KnownSetting "ignore_drop_queries_probability" Float instance KnownSetting "traverse_shadow_remote_data_paths" Bool instance KnownSetting "geo_distance_returns_float64_on_float64_arguments" Bool instance KnownSetting "allow_get_client_http_header" Bool @@ -900,7 +1004,6 @@ instance KnownSetting "allow_dynamic_type_in_join_keys" Bool instance KnownSetting "cast_string_to_variant_use_inference" Bool instance KnownSetting "enable_blob_storage_log" Bool -instance KnownSetting "use_json_alias_for_old_object_type" Bool instance KnownSetting "allow_create_index_without_type" Bool instance KnownSetting "create_index_ignore_unique" Bool instance KnownSetting "print_pretty_type_names" Bool @@ -909,9 +1012,17 @@ instance KnownSetting "default_view_definer" ChString instance KnownSetting "cache_warmer_threads" UInt64 instance KnownSetting "use_async_executor_for_materialized_views" Bool +instance KnownSetting "ignore_cold_parts_seconds" Int64 instance KnownSetting "short_circuit_function_evaluation_for_nulls" Bool +instance KnownSetting "short_circuit_function_evaluation_for_nulls_threshold" Double +instance KnownSetting "prefer_warmed_unmerged_parts_seconds" Int64 +instance KnownSetting "iceberg_timestamp_ms" Int64 +instance KnownSetting "iceberg_snapshot_id" Int64 instance KnownSetting "show_data_lake_catalogs_in_system_tables" Bool instance KnownSetting "delta_lake_enable_expression_visitor_logging" Bool +instance KnownSetting "delta_lake_snapshot_version" Int64 +instance KnownSetting "delta_lake_snapshot_start_version" Int64 +instance KnownSetting "delta_lake_snapshot_end_version" Int64 instance KnownSetting "delta_lake_throw_on_engine_predicate_error" Bool instance KnownSetting "delta_lake_enable_engine_predicate" Bool instance KnownSetting "allow_experimental_delta_lake_writes" Bool @@ -923,6 +1034,8 @@ instance KnownSetting "restore_replace_external_engines_to_null" Bool instance KnownSetting "restore_replace_external_table_functions_to_null" Bool instance KnownSetting "restore_replace_external_dictionary_source_to_null" Bool +instance KnownSetting "automatic_parallel_replicas_mode" UInt64 +instance KnownSetting "automatic_parallel_replicas_min_bytes_per_replica" UInt64 instance KnownSetting "parallel_replicas_count" UInt64 instance KnownSetting "parallel_replica_offset" UInt64 instance KnownSetting "parallel_replicas_custom_key" ChString @@ -940,6 +1053,7 @@ instance KnownSetting "parallel_replicas_only_with_analyzer" Bool instance KnownSetting "parallel_replicas_insert_select_local_pipeline" Bool instance KnownSetting "parallel_replicas_for_cluster_engines" Bool +instance KnownSetting "parallel_replicas_allow_materialized_views" Bool instance KnownSetting "analyzer_compatibility_join_using_top_level_identifier" Bool instance KnownSetting "analyzer_compatibility_allow_compound_identifiers_in_unflatten_nested" Bool instance KnownSetting "create_if_not_exists" Bool @@ -956,6 +1070,7 @@ instance KnownSetting "shared_merge_tree_sync_parts_on_partition_operations" Bool instance KnownSetting "implicit_table_at_top_level" ChString instance KnownSetting "allow_general_join_planning" Bool +instance KnownSetting "cluster_table_function_buckets_batch_size" UInt64 instance KnownSetting "merge_table_max_tables_to_look_for_schema_inference" UInt64 instance KnownSetting "validate_enum_literals_in_operators" Bool instance KnownSetting "max_autoincrement_series" UInt64 @@ -965,20 +1080,29 @@ instance KnownSetting "allow_archive_path_syntax" Bool instance KnownSetting "iceberg_insert_max_rows_in_data_file" UInt64 instance KnownSetting "iceberg_insert_max_bytes_in_data_file" UInt64 +instance KnownSetting "iceberg_insert_max_partitions" UInt64 +instance KnownSetting "min_os_cpu_wait_time_ratio_to_throw" Float +instance KnownSetting "max_os_cpu_wait_time_ratio_to_throw" Float instance KnownSetting "enable_producing_buckets_out_of_order_in_aggregation" Bool instance KnownSetting "enable_parallel_blocks_marshalling" Bool instance KnownSetting "min_outstreams_per_resize_after_split" UInt64 instance KnownSetting "enable_add_distinct_to_in_subqueries" Bool instance KnownSetting "function_date_trunc_return_type_behavior" UInt64 +instance KnownSetting "query_plan_remove_unused_columns" Bool instance KnownSetting "jemalloc_enable_profiler" Bool instance KnownSetting "jemalloc_collect_profile_samples_in_trace_log" Bool +instance KnownSetting "show_processlist_include_internal" Bool instance KnownSetting "use_roaring_bitmap_iceberg_positional_deletes" Bool instance KnownSetting "inject_random_order_for_select_without_order_by" Bool +instance KnownSetting "optimize_const_name_size" Int64 +instance KnownSetting "serialize_string_in_memory_with_zero_byte" Bool +instance KnownSetting "s3_path_filter_limit" UInt64 +instance KnownSetting "ignore_on_cluster_for_replicated_database" Bool +instance KnownSetting "archive_adaptive_buffer_max_size_bytes" UInt64 instance KnownSetting "allow_experimental_materialized_postgresql_table" Bool instance KnownSetting "allow_experimental_funnel_functions" Bool instance KnownSetting "allow_experimental_nlp_functions" Bool instance KnownSetting "allow_experimental_hash_functions" Bool -instance KnownSetting "allow_experimental_object_type" Bool instance KnownSetting "allow_experimental_time_series_table" Bool instance KnownSetting "allow_experimental_codecs" Bool instance KnownSetting "throw_on_unsupported_query_inside_transaction" Bool @@ -987,14 +1111,15 @@ instance KnownSetting "join_to_sort_maximum_table_rows" UInt64 instance KnownSetting "allow_experimental_join_right_table_sorting" Bool instance KnownSetting "use_statistics_cache" Bool -instance KnownSetting "allow_experimental_full_text_index" Bool instance KnownSetting "query_plan_direct_read_from_text_index" Bool +instance KnownSetting "query_plan_text_index_add_hint" Bool +instance KnownSetting "text_index_hint_max_selectivity" Float instance KnownSetting "use_text_index_dictionary_cache" Bool instance KnownSetting "use_text_index_header_cache" Bool +instance KnownSetting "use_text_index_postings_cache" Bool instance KnownSetting "allow_experimental_window_view" Bool instance KnownSetting "stop_refreshable_materialized_views_on_startup" Bool instance KnownSetting "allow_experimental_database_materialized_postgresql" Bool -instance KnownSetting "allow_experimental_qbit_type" Bool instance KnownSetting "allow_experimental_query_deduplication" Bool instance KnownSetting "allow_experimental_database_hms_catalog" Bool instance KnownSetting "allow_experimental_kusto_dialect" Bool @@ -1007,7 +1132,6 @@ instance KnownSetting "iceberg_metadata_compression_method" ChString instance KnownSetting "make_distributed_plan" Bool instance KnownSetting "distributed_plan_execute_locally" Bool -instance KnownSetting "distributed_plan_default_shuffle_join_bucket_count" UInt64 instance KnownSetting "distributed_plan_default_reader_bucket_count" UInt64 instance KnownSetting "distributed_plan_optimize_exchanges" Bool instance KnownSetting "distributed_plan_force_exchange_kind" ChString @@ -1020,14 +1144,22 @@ instance KnownSetting "join_runtime_filter_exact_values_limit" UInt64 instance KnownSetting "join_runtime_bloom_filter_bytes" UInt64 instance KnownSetting "join_runtime_bloom_filter_hash_functions" UInt64 +instance KnownSetting "join_runtime_filter_pass_ratio_threshold_for_disabling" Double +instance KnownSetting "join_runtime_filter_blocks_to_skip_before_reenabling" UInt64 +instance KnownSetting "join_runtime_bloom_filter_max_ratio_of_set_bits" Double instance KnownSetting "rewrite_in_to_join" Bool instance KnownSetting "promql_database" ChString instance KnownSetting "promql_table" ChString +instance KnownSetting "allow_experimental_alias_table_engine" Bool +instance KnownSetting "use_paimon_partition_pruning" Bool +instance KnownSetting "allow_experimental_object_storage_queue_hive_partitioning" Bool +instance KnownSetting "allow_experimental_database_paimon_rest_catalog" Bool settingsMap :: [(ChString, SettingSerializer)] settingsMap = [ mkSettingSerializer @"min_compress_block_size", mkSettingSerializer @"max_compress_block_size", + mkSettingSerializer @"max_insert_block_size_bytes", mkSettingSerializer @"min_insert_block_size_rows", mkSettingSerializer @"min_insert_block_size_bytes", mkSettingSerializer @"min_insert_block_size_rows_for_materialized_views", @@ -1115,6 +1247,7 @@ mkSettingSerializer @"azure_request_timeout_ms", mkSettingSerializer @"azure_connect_timeout_ms", mkSettingSerializer @"s3_validate_request_settings", + mkSettingSerializer @"compatibility_s3_presigned_url_query_in_path", mkSettingSerializer @"s3_disable_checksum", mkSettingSerializer @"s3_request_timeout_ms", mkSettingSerializer @"s3_connect_timeout_ms", @@ -1122,6 +1255,7 @@ mkSettingSerializer @"s3queue_default_zookeeper_path", mkSettingSerializer @"s3queue_migrate_old_metadata_to_buckets", mkSettingSerializer @"s3queue_enable_logging_to_s3queue_log", + mkSettingSerializer @"s3queue_keeper_fault_injection_probability", mkSettingSerializer @"hdfs_replication", mkSettingSerializer @"hdfs_truncate_on_insert", mkSettingSerializer @"hdfs_create_new_file_on_insert", @@ -1146,8 +1280,10 @@ mkSettingSerializer @"enable_multiple_prewhere_read_steps", mkSettingSerializer @"move_primary_key_columns_to_end_of_prewhere", mkSettingSerializer @"allow_reorder_prewhere_conditions", + mkSettingSerializer @"replication_wait_for_inactive_replica_timeout", mkSettingSerializer @"alter_move_to_space_execute_async", mkSettingSerializer @"load_balancing_first_offset", + mkSettingSerializer @"totals_auto_threshold", mkSettingSerializer @"allow_suspicious_low_cardinality_types", mkSettingSerializer @"allow_suspicious_fixed_string_types", mkSettingSerializer @"allow_suspicious_indices", @@ -1156,8 +1292,7 @@ mkSettingSerializer @"allow_suspicious_primary_key", mkSettingSerializer @"allow_suspicious_types_in_group_by", mkSettingSerializer @"allow_suspicious_types_in_order_by", - mkSettingSerializer @"allow_not_comparable_types_in_order_by", - mkSettingSerializer @"allow_not_comparable_types_in_comparison_functions", + mkSettingSerializer @"use_variant_default_implementation_for_comparisons", mkSettingSerializer @"compile_expressions", mkSettingSerializer @"min_count_to_compile_expression", mkSettingSerializer @"compile_aggregate_expressions", @@ -1170,6 +1305,7 @@ mkSettingSerializer @"aggregation_memory_efficient_merge_threads", mkSettingSerializer @"enable_memory_bound_merging_of_aggregation_results", mkSettingSerializer @"enable_positional_arguments", + mkSettingSerializer @"enable_positional_arguments_for_projections", mkSettingSerializer @"enable_extended_results_for_datetime_functions", mkSettingSerializer @"allow_nonconst_timezone_arguments", mkSettingSerializer @"use_legacy_to_time", @@ -1187,10 +1323,6 @@ mkSettingSerializer @"force_optimize_skip_unused_shards", mkSettingSerializer @"optimize_skip_unused_shards_nesting", mkSettingSerializer @"force_optimize_skip_unused_shards_nesting", - mkSettingSerializer @"input_format_parallel_parsing", - mkSettingSerializer @"output_format_parallel_formatting", - mkSettingSerializer @"output_format_compression_level", - mkSettingSerializer @"output_format_compression_zstd_window_log", mkSettingSerializer @"allow_special_serialization_kinds_in_output_formats", mkSettingSerializer @"enable_parsing_to_custom_serialization", mkSettingSerializer @"merge_tree_use_v1_object_and_dynamic_serialization", @@ -1206,6 +1338,8 @@ mkSettingSerializer @"do_not_merge_across_partitions_select_final", mkSettingSerializer @"split_parts_ranges_into_intersecting_and_non_intersecting_final", mkSettingSerializer @"split_intersecting_parts_ranges_into_layers_final", + mkSettingSerializer @"apply_row_policy_after_final", + mkSettingSerializer @"apply_prewhere_after_final", mkSettingSerializer @"mysql_max_rows_to_insert", mkSettingSerializer @"mysql_map_string_to_text_in_show_columns", mkSettingSerializer @"mysql_map_fixed_string_to_text_in_show_columns", @@ -1215,11 +1349,16 @@ mkSettingSerializer @"min_bytes_to_use_mmap_io", mkSettingSerializer @"checksum_on_read", mkSettingSerializer @"force_index_by_date", + mkSettingSerializer @"use_primary_key", mkSettingSerializer @"force_primary_key", mkSettingSerializer @"use_skip_indexes", mkSettingSerializer @"use_skip_indexes_if_final", mkSettingSerializer @"use_skip_indexes_if_final_exact_mode", mkSettingSerializer @"use_skip_indexes_on_data_read", + mkSettingSerializer @"use_skip_indexes_for_disjunctions", + mkSettingSerializer @"use_skip_indexes_for_top_k", + mkSettingSerializer @"use_top_k_dynamic_filtering", + mkSettingSerializer @"query_plan_max_limit_for_top_k_optimization", mkSettingSerializer @"materialize_skip_indexes_on_insert", mkSettingSerializer @"exclude_materialize_skip_indexes_on_insert", mkSettingSerializer @"text_index_use_bloom_filter", @@ -1228,11 +1367,16 @@ mkSettingSerializer @"ignore_data_skipping_indices", mkSettingSerializer @"force_data_skipping_indices", mkSettingSerializer @"secondary_indices_enable_bulk_filtering", + mkSettingSerializer @"max_streams_to_max_threads_ratio", + mkSettingSerializer @"max_streams_multiplier_for_merge_tables", mkSettingSerializer @"network_compression_method", + mkSettingSerializer @"network_zstd_compression_level", + mkSettingSerializer @"zstd_window_log_max", mkSettingSerializer @"priority", mkSettingSerializer @"log_queries", mkSettingSerializer @"log_formatted_queries", mkSettingSerializer @"log_queries_cut_to_length", + mkSettingSerializer @"log_queries_probability", mkSettingSerializer @"log_processors_profiles", mkSettingSerializer @"max_concurrent_queries_for_all_users", mkSettingSerializer @"max_concurrent_queries_for_user", @@ -1245,12 +1389,16 @@ mkSettingSerializer @"read_backoff_max_throughput", mkSettingSerializer @"read_backoff_min_events", mkSettingSerializer @"read_backoff_min_concurrency", + mkSettingSerializer @"memory_tracker_fault_probability", + mkSettingSerializer @"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", mkSettingSerializer @"enable_http_compression", + mkSettingSerializer @"http_zlib_compression_level", mkSettingSerializer @"http_native_compression_disable_checksumming_on_decompress", mkSettingSerializer @"count_distinct_implementation", mkSettingSerializer @"add_http_cors_header", mkSettingSerializer @"max_http_get_redirects", mkSettingSerializer @"use_client_time_zone", + mkSettingSerializer @"send_profile_events", mkSettingSerializer @"send_progress_in_http_headers", mkSettingSerializer @"http_headers_progress_interval_ms", mkSettingSerializer @"http_wait_end_of_query", @@ -1273,7 +1421,9 @@ mkSettingSerializer @"parts_to_throw_insert", mkSettingSerializer @"number_of_mutations_to_delay", mkSettingSerializer @"number_of_mutations_to_throw", + mkSettingSerializer @"distributed_ddl_task_timeout", mkSettingSerializer @"min_free_disk_bytes_to_perform_insert", + mkSettingSerializer @"min_free_disk_ratio_to_perform_insert", mkSettingSerializer @"final", mkSettingSerializer @"partial_result_on_first_cancel", mkSettingSerializer @"ignore_on_cluster_for_replicated_udf_queries", @@ -1296,12 +1446,11 @@ mkSettingSerializer @"empty_result_for_aggregation_by_constant_keys_on_empty_set", mkSettingSerializer @"allow_distributed_ddl", mkSettingSerializer @"allow_suspicious_codecs", - mkSettingSerializer @"enable_zstd_qat_codec", - mkSettingSerializer @"enable_deflate_qpl_codec", mkSettingSerializer @"query_profiler_real_time_period_ns", mkSettingSerializer @"query_profiler_cpu_time_period_ns", mkSettingSerializer @"metrics_perf_events_enabled", mkSettingSerializer @"metrics_perf_events_list", + mkSettingSerializer @"opentelemetry_start_trace_probability", mkSettingSerializer @"opentelemetry_trace_processors", mkSettingSerializer @"opentelemetry_trace_cpu_scheduling", mkSettingSerializer @"prefer_column_name_to_alias", @@ -1314,11 +1463,14 @@ mkSettingSerializer @"max_bytes_to_read_leaf", mkSettingSerializer @"max_rows_to_group_by", mkSettingSerializer @"max_bytes_before_external_group_by", + mkSettingSerializer @"max_bytes_ratio_before_external_group_by", mkSettingSerializer @"max_rows_to_sort", mkSettingSerializer @"max_bytes_to_sort", mkSettingSerializer @"prefer_external_sort_block_bytes", mkSettingSerializer @"max_bytes_before_external_sort", + mkSettingSerializer @"max_bytes_ratio_before_external_sort", mkSettingSerializer @"max_bytes_before_remerge_sort", + mkSettingSerializer @"remerge_sort_lowered_memory_bytes_ratio", mkSettingSerializer @"max_result_rows", mkSettingSerializer @"max_result_bytes", mkSettingSerializer @"min_execution_speed", @@ -1337,9 +1489,12 @@ mkSettingSerializer @"readonly", mkSettingSerializer @"max_rows_in_set", mkSettingSerializer @"max_bytes_in_set", + mkSettingSerializer @"exact_rows_before_limit", + mkSettingSerializer @"rows_before_aggregation", mkSettingSerializer @"max_rows_in_join", mkSettingSerializer @"max_bytes_in_join", mkSettingSerializer @"join_any_take_last_row", + mkSettingSerializer @"cross_to_inner_join_rewrite", mkSettingSerializer @"cross_join_min_rows_to_compress", mkSettingSerializer @"cross_join_min_bytes_to_compress", mkSettingSerializer @"default_max_bytes_in_join", @@ -1349,7 +1504,6 @@ mkSettingSerializer @"max_rows_in_set_to_optimize_join", mkSettingSerializer @"compatibility_ignore_collation_in_create_table", mkSettingSerializer @"temporary_files_codec", - mkSettingSerializer @"temporary_files_buffer_size", mkSettingSerializer @"max_rows_to_transfer", mkSettingSerializer @"max_bytes_to_transfer", mkSettingSerializer @"max_rows_in_distinct", @@ -1359,10 +1513,13 @@ mkSettingSerializer @"max_memory_usage_for_user", mkSettingSerializer @"memory_overcommit_ratio_denominator_for_user", mkSettingSerializer @"max_untracked_memory", + mkSettingSerializer @"max_reverse_dictionary_lookup_cache_size_bytes", mkSettingSerializer @"memory_profiler_step", + mkSettingSerializer @"memory_profiler_sample_probability", mkSettingSerializer @"memory_profiler_sample_min_allocation_size", mkSettingSerializer @"memory_profiler_sample_max_allocation_size", mkSettingSerializer @"trace_profile_events", + mkSettingSerializer @"trace_profile_events_list", mkSettingSerializer @"memory_usage_overcommit_max_wait_microseconds", mkSettingSerializer @"max_network_bandwidth", mkSettingSerializer @"max_network_bytes", @@ -1380,10 +1537,12 @@ mkSettingSerializer @"backup_restore_keeper_value_max_size", mkSettingSerializer @"backup_restore_batch_size_for_keeper_multi", mkSettingSerializer @"backup_restore_batch_size_for_keeper_multiread", + mkSettingSerializer @"backup_restore_keeper_fault_injection_probability", mkSettingSerializer @"backup_restore_keeper_fault_injection_seed", mkSettingSerializer @"backup_restore_s3_retry_attempts", mkSettingSerializer @"backup_restore_s3_retry_initial_backoff_ms", mkSettingSerializer @"backup_restore_s3_retry_max_backoff_ms", + mkSettingSerializer @"backup_restore_s3_retry_jitter_factor", mkSettingSerializer @"max_backup_bandwidth", mkSettingSerializer @"restore_replicated_merge_tree_to_shared_merge_tree", mkSettingSerializer @"log_profile_events", @@ -1391,6 +1550,11 @@ mkSettingSerializer @"log_query_threads", mkSettingSerializer @"log_query_views", mkSettingSerializer @"log_comment", + mkSettingSerializer @"query_metric_log_interval", + mkSettingSerializer @"regexp_dict_allow_hyperscan", + mkSettingSerializer @"regexp_dict_flag_case_insensitive", + mkSettingSerializer @"regexp_dict_flag_dotall", + mkSettingSerializer @"dictionary_use_async_executor", mkSettingSerializer @"send_logs_source_regexp", mkSettingSerializer @"enable_optimize_predicate_expression", mkSettingSerializer @"enable_optimize_predicate_expression_to_final_subquery", @@ -1438,6 +1602,7 @@ mkSettingSerializer @"h3togeo_lon_lat_result_order", mkSettingSerializer @"max_partitions_per_insert_block", mkSettingSerializer @"throw_on_max_partitions_per_insert_block", + mkSettingSerializer @"max_partitions_to_read", mkSettingSerializer @"check_query_single_value_result", mkSettingSerializer @"allow_drop_detached", mkSettingSerializer @"max_parts_to_move", @@ -1448,6 +1613,7 @@ mkSettingSerializer @"postgresql_connection_pool_wait_timeout", mkSettingSerializer @"postgresql_connection_pool_retries", mkSettingSerializer @"postgresql_connection_pool_auto_close_connection", + mkSettingSerializer @"postgresql_fault_injection_probability", mkSettingSerializer @"glob_expansion_max_elements", mkSettingSerializer @"odbc_bridge_connection_pool_size", mkSettingSerializer @"odbc_bridge_use_connection_pooling", @@ -1484,7 +1650,6 @@ mkSettingSerializer @"normalize_function_names", mkSettingSerializer @"enable_early_constant_folding", mkSettingSerializer @"deduplicate_blocks_in_dependent_materialized_views", - mkSettingSerializer @"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", mkSettingSerializer @"materialized_views_ignore_errors", mkSettingSerializer @"ignore_materialized_views_with_dropped_target_table", mkSettingSerializer @"allow_materialized_view_with_bad_select", @@ -1526,7 +1691,6 @@ mkSettingSerializer @"async_socket_for_remote", mkSettingSerializer @"async_query_sending_for_remote", mkSettingSerializer @"insert_null_as_default", - mkSettingSerializer @"describe_extend_object_types", mkSettingSerializer @"describe_include_subcolumns", mkSettingSerializer @"describe_include_virtual_columns", mkSettingSerializer @"describe_compact_output", @@ -1535,6 +1699,7 @@ mkSettingSerializer @"mutations_execute_nondeterministic_on_initiator", mkSettingSerializer @"mutations_execute_subqueries_on_initiator", mkSettingSerializer @"mutations_max_literal_size_to_replace", + mkSettingSerializer @"create_replicated_merge_tree_fault_injection_probability", mkSettingSerializer @"delta_lake_log_metadata", mkSettingSerializer @"iceberg_delete_data_on_drop", mkSettingSerializer @"use_iceberg_metadata_files_cache", @@ -1564,6 +1729,7 @@ mkSettingSerializer @"collect_hash_table_stats_during_aggregation", mkSettingSerializer @"max_size_to_preallocate_for_aggregation", mkSettingSerializer @"collect_hash_table_stats_during_joins", + mkSettingSerializer @"use_hash_table_stats_for_join_reordering", mkSettingSerializer @"max_size_to_preallocate_for_joins", mkSettingSerializer @"kafka_disable_num_consumers_limit", mkSettingSerializer @"allow_experimental_kafka_offsets_storage_in_keeper", @@ -1571,6 +1737,7 @@ mkSettingSerializer @"allow_aggregate_partitions_independently", mkSettingSerializer @"force_aggregate_partitions_independently", mkSettingSerializer @"max_number_of_partitions_for_independent_aggregation", + mkSettingSerializer @"min_hit_rate_to_use_consecutive_keys_optimization", mkSettingSerializer @"engine_file_empty_if_not_exists", mkSettingSerializer @"engine_file_truncate_on_insert", mkSettingSerializer @"engine_file_allow_create_multiple_files", @@ -1585,6 +1752,7 @@ mkSettingSerializer @"database_replicated_allow_replicated_engine_arguments", mkSettingSerializer @"database_replicated_allow_explicit_uuid", mkSettingSerializer @"database_replicated_allow_heavy_create", + mkSettingSerializer @"database_shared_drop_table_delay_seconds", mkSettingSerializer @"cloud_mode", mkSettingSerializer @"cloud_mode_engine", mkSettingSerializer @"cloud_mode_database_engine", @@ -1617,6 +1785,7 @@ mkSettingSerializer @"query_plan_reuse_storage_ordering_for_window_functions", mkSettingSerializer @"query_plan_lift_up_union", mkSettingSerializer @"query_plan_read_in_order", + mkSettingSerializer @"query_plan_read_in_order_through_join", mkSettingSerializer @"query_plan_aggregation_in_order", mkSettingSerializer @"query_plan_remove_redundant_sorting", mkSettingSerializer @"query_plan_remove_redundant_distinct", @@ -1627,6 +1796,7 @@ mkSettingSerializer @"enable_lazy_columns_replication", mkSettingSerializer @"serialize_query_plan", mkSettingSerializer @"correlated_subqueries_substitute_equivalent_expressions", + mkSettingSerializer @"correlated_subqueries_use_in_memory_buffer", mkSettingSerializer @"optimize_qbit_distance_function_reads", mkSettingSerializer @"regexp_max_matches_per_row", mkSettingSerializer @"limit", @@ -1638,6 +1808,7 @@ mkSettingSerializer @"remote_filesystem_read_method", mkSettingSerializer @"local_filesystem_read_prefetch", mkSettingSerializer @"remote_filesystem_read_prefetch", + mkSettingSerializer @"read_priority", mkSettingSerializer @"merge_tree_min_rows_for_concurrent_read_for_remote_filesystem", mkSettingSerializer @"merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem", mkSettingSerializer @"remote_read_min_bytes_for_seek", @@ -1649,9 +1820,12 @@ mkSettingSerializer @"async_insert_max_data_size", mkSettingSerializer @"async_insert_max_query_number", mkSettingSerializer @"async_insert_use_adaptive_busy_timeout", + mkSettingSerializer @"async_insert_busy_timeout_increase_rate", + mkSettingSerializer @"async_insert_busy_timeout_decrease_rate", mkSettingSerializer @"remote_fs_read_max_backoff_ms", mkSettingSerializer @"remote_fs_read_backoff_max_tries", mkSettingSerializer @"cluster_function_process_archive_on_multiple_nodes", + mkSettingSerializer @"max_streams_for_files_processing_in_cluster_functions", mkSettingSerializer @"enable_filesystem_cache", mkSettingSerializer @"filesystem_cache_name", mkSettingSerializer @"enable_filesystem_cache_on_write_operations", @@ -1728,11 +1902,16 @@ mkSettingSerializer @"distributed_cache_receive_timeout_ms", mkSettingSerializer @"distributed_cache_send_timeout_ms", mkSettingSerializer @"distributed_cache_tcp_keep_alive_timeout_ms", + mkSettingSerializer @"distributed_cache_use_clients_cache_for_write", + mkSettingSerializer @"distributed_cache_use_clients_cache_for_read", + mkSettingSerializer @"distributed_cache_file_cache_name", + mkSettingSerializer @"filesystem_cache_allow_background_download", mkSettingSerializer @"filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage", mkSettingSerializer @"filesystem_cache_enable_background_download_during_fetch", mkSettingSerializer @"parallelize_output_from_storages", mkSettingSerializer @"insert_deduplication_token", mkSettingSerializer @"count_distinct_optimization", + mkSettingSerializer @"optimize_inverse_dictionary_lookup", mkSettingSerializer @"throw_if_no_data_to_insert", mkSettingSerializer @"compatibility_ignore_auto_increment_in_create_table", mkSettingSerializer @"multiple_joins_try_to_keep_original_names", @@ -1743,6 +1922,7 @@ mkSettingSerializer @"insert_keeper_max_retries", mkSettingSerializer @"insert_keeper_retry_initial_backoff_ms", mkSettingSerializer @"insert_keeper_retry_max_backoff_ms", + mkSettingSerializer @"insert_keeper_fault_injection_probability", mkSettingSerializer @"insert_keeper_fault_injection_seed", mkSettingSerializer @"force_aggregation_in_order", mkSettingSerializer @"http_max_request_param_data_size", @@ -1752,6 +1932,7 @@ mkSettingSerializer @"optimize_uniq_to_count", mkSettingSerializer @"use_variant_as_common_type", mkSettingSerializer @"enable_order_by_all", + mkSettingSerializer @"ignore_drop_queries_probability", mkSettingSerializer @"traverse_shadow_remote_data_paths", mkSettingSerializer @"geo_distance_returns_float64_on_float64_arguments", mkSettingSerializer @"allow_get_client_http_header", @@ -1759,7 +1940,6 @@ mkSettingSerializer @"allow_dynamic_type_in_join_keys", mkSettingSerializer @"cast_string_to_variant_use_inference", mkSettingSerializer @"enable_blob_storage_log", - mkSettingSerializer @"use_json_alias_for_old_object_type", mkSettingSerializer @"allow_create_index_without_type", mkSettingSerializer @"create_index_ignore_unique", mkSettingSerializer @"print_pretty_type_names", @@ -1768,9 +1948,17 @@ mkSettingSerializer @"default_view_definer", mkSettingSerializer @"cache_warmer_threads", mkSettingSerializer @"use_async_executor_for_materialized_views", + mkSettingSerializer @"ignore_cold_parts_seconds", mkSettingSerializer @"short_circuit_function_evaluation_for_nulls", + mkSettingSerializer @"short_circuit_function_evaluation_for_nulls_threshold", + mkSettingSerializer @"prefer_warmed_unmerged_parts_seconds", + mkSettingSerializer @"iceberg_timestamp_ms", + mkSettingSerializer @"iceberg_snapshot_id", mkSettingSerializer @"show_data_lake_catalogs_in_system_tables", mkSettingSerializer @"delta_lake_enable_expression_visitor_logging", + mkSettingSerializer @"delta_lake_snapshot_version", + mkSettingSerializer @"delta_lake_snapshot_start_version", + mkSettingSerializer @"delta_lake_snapshot_end_version", mkSettingSerializer @"delta_lake_throw_on_engine_predicate_error", mkSettingSerializer @"delta_lake_enable_engine_predicate", mkSettingSerializer @"allow_experimental_delta_lake_writes", @@ -1782,6 +1970,8 @@ mkSettingSerializer @"restore_replace_external_engines_to_null", mkSettingSerializer @"restore_replace_external_table_functions_to_null", mkSettingSerializer @"restore_replace_external_dictionary_source_to_null", + mkSettingSerializer @"automatic_parallel_replicas_mode", + mkSettingSerializer @"automatic_parallel_replicas_min_bytes_per_replica", mkSettingSerializer @"parallel_replicas_count", mkSettingSerializer @"parallel_replica_offset", mkSettingSerializer @"parallel_replicas_custom_key", @@ -1799,6 +1989,7 @@ mkSettingSerializer @"parallel_replicas_only_with_analyzer", mkSettingSerializer @"parallel_replicas_insert_select_local_pipeline", mkSettingSerializer @"parallel_replicas_for_cluster_engines", + mkSettingSerializer @"parallel_replicas_allow_materialized_views", mkSettingSerializer @"analyzer_compatibility_join_using_top_level_identifier", mkSettingSerializer @"analyzer_compatibility_allow_compound_identifiers_in_unflatten_nested", mkSettingSerializer @"create_if_not_exists", @@ -1815,6 +2006,7 @@ mkSettingSerializer @"shared_merge_tree_sync_parts_on_partition_operations", mkSettingSerializer @"implicit_table_at_top_level", mkSettingSerializer @"allow_general_join_planning", + mkSettingSerializer @"cluster_table_function_buckets_batch_size", mkSettingSerializer @"merge_table_max_tables_to_look_for_schema_inference", mkSettingSerializer @"validate_enum_literals_in_operators", mkSettingSerializer @"max_autoincrement_series", @@ -1824,20 +2016,29 @@ mkSettingSerializer @"allow_archive_path_syntax", mkSettingSerializer @"iceberg_insert_max_rows_in_data_file", mkSettingSerializer @"iceberg_insert_max_bytes_in_data_file", + mkSettingSerializer @"iceberg_insert_max_partitions", + mkSettingSerializer @"min_os_cpu_wait_time_ratio_to_throw", + mkSettingSerializer @"max_os_cpu_wait_time_ratio_to_throw", mkSettingSerializer @"enable_producing_buckets_out_of_order_in_aggregation", mkSettingSerializer @"enable_parallel_blocks_marshalling", mkSettingSerializer @"min_outstreams_per_resize_after_split", mkSettingSerializer @"enable_add_distinct_to_in_subqueries", mkSettingSerializer @"function_date_trunc_return_type_behavior", + mkSettingSerializer @"query_plan_remove_unused_columns", mkSettingSerializer @"jemalloc_enable_profiler", mkSettingSerializer @"jemalloc_collect_profile_samples_in_trace_log", + mkSettingSerializer @"show_processlist_include_internal", mkSettingSerializer @"use_roaring_bitmap_iceberg_positional_deletes", mkSettingSerializer @"inject_random_order_for_select_without_order_by", + mkSettingSerializer @"optimize_const_name_size", + mkSettingSerializer @"serialize_string_in_memory_with_zero_byte", + mkSettingSerializer @"s3_path_filter_limit", + mkSettingSerializer @"ignore_on_cluster_for_replicated_database", + mkSettingSerializer @"archive_adaptive_buffer_max_size_bytes", mkSettingSerializer @"allow_experimental_materialized_postgresql_table", mkSettingSerializer @"allow_experimental_funnel_functions", mkSettingSerializer @"allow_experimental_nlp_functions", mkSettingSerializer @"allow_experimental_hash_functions", - mkSettingSerializer @"allow_experimental_object_type", mkSettingSerializer @"allow_experimental_time_series_table", mkSettingSerializer @"allow_experimental_codecs", mkSettingSerializer @"throw_on_unsupported_query_inside_transaction", @@ -1846,14 +2047,15 @@ mkSettingSerializer @"join_to_sort_maximum_table_rows", mkSettingSerializer @"allow_experimental_join_right_table_sorting", mkSettingSerializer @"use_statistics_cache", - mkSettingSerializer @"allow_experimental_full_text_index", mkSettingSerializer @"query_plan_direct_read_from_text_index", + mkSettingSerializer @"query_plan_text_index_add_hint", + mkSettingSerializer @"text_index_hint_max_selectivity", mkSettingSerializer @"use_text_index_dictionary_cache", mkSettingSerializer @"use_text_index_header_cache", + mkSettingSerializer @"use_text_index_postings_cache", mkSettingSerializer @"allow_experimental_window_view", mkSettingSerializer @"stop_refreshable_materialized_views_on_startup", mkSettingSerializer @"allow_experimental_database_materialized_postgresql", - mkSettingSerializer @"allow_experimental_qbit_type", mkSettingSerializer @"allow_experimental_query_deduplication", mkSettingSerializer @"allow_experimental_database_hms_catalog", mkSettingSerializer @"allow_experimental_kusto_dialect", @@ -1866,7 +2068,6 @@ mkSettingSerializer @"iceberg_metadata_compression_method", mkSettingSerializer @"make_distributed_plan", mkSettingSerializer @"distributed_plan_execute_locally", - mkSettingSerializer @"distributed_plan_default_shuffle_join_bucket_count", mkSettingSerializer @"distributed_plan_default_reader_bucket_count", mkSettingSerializer @"distributed_plan_optimize_exchanges", mkSettingSerializer @"distributed_plan_force_exchange_kind", @@ -1879,7 +2080,14 @@ mkSettingSerializer @"join_runtime_filter_exact_values_limit", mkSettingSerializer @"join_runtime_bloom_filter_bytes", mkSettingSerializer @"join_runtime_bloom_filter_hash_functions", + mkSettingSerializer @"join_runtime_filter_pass_ratio_threshold_for_disabling", + mkSettingSerializer @"join_runtime_filter_blocks_to_skip_before_reenabling", + mkSettingSerializer @"join_runtime_bloom_filter_max_ratio_of_set_bits", mkSettingSerializer @"rewrite_in_to_join", mkSettingSerializer @"promql_database", - mkSettingSerializer @"promql_table" + mkSettingSerializer @"promql_table", + mkSettingSerializer @"allow_experimental_alias_table_engine", + mkSettingSerializer @"use_paimon_partition_pruning", + mkSettingSerializer @"allow_experimental_object_storage_queue_hive_partitioning", + mkSettingSerializer @"allow_experimental_database_paimon_rest_catalog" ] diff --git a/ClickHaskell/Statements.hs b/ClickHaskell/Statements.hs --- a/ClickHaskell/Statements.hs +++ b/ClickHaskell/Statements.hs @@ -2,7 +2,8 @@ -- Internal import ClickHaskell.Primitive -import ClickHaskell.Protocol.Settings (DbSettings (..)) +import ClickHaskell.Protocol.Settings (DbSettings (..), emptySettings) +import ClickHaskell.Protocol.Client (QueryParameters (..), emptyParameters) -- GHC included import Data.ByteString.Builder (Builder, byteString) @@ -24,25 +25,29 @@ Wrapper for settings passing -} passSettings :: (DbSettings -> DbSettings) -> statement -> statement + passParameters :: (QueryParameters -> QueryParameters) -> statement -> statement instance Statement (Select cols output) where - passSettings pass (MkSelect mkQuery dbSettings) = MkSelect mkQuery (pass dbSettings) + passSettings pass (MkSelect mkQuery dbSettings parameters) = MkSelect mkQuery (pass dbSettings) parameters + passParameters pass (MkSelect mkQuery dbSettings parameters) = MkSelect mkQuery dbSettings (pass parameters) instance Statement (Insert cols input) where - passSettings pass (MkInsert mkQuery dbSettings) = MkInsert mkQuery (pass dbSettings) + passSettings pass (MkInsert mkQuery dbSettings parameters) = MkInsert mkQuery (pass dbSettings) parameters + passParameters pass (MkInsert mkQuery dbSettings parameters) = MkInsert mkQuery dbSettings (pass parameters) instance Statement (Command) where - passSettings pass (MkCommand query dbSettings) = MkCommand query (pass dbSettings) + passSettings pass (MkCommand mkQuery dbSettings parameters) = MkCommand mkQuery (pass dbSettings) parameters + passParameters pass (MkCommand mkQuery dbSettings parameters) = MkCommand mkQuery dbSettings (pass parameters) -- ** Command data Command where - MkCommand :: ChString -> DbSettings -> Command + MkCommand :: ChString -> DbSettings -> QueryParameters -> Command instance IsString Command where - fromString str = MkCommand (toChType str) (MkDbSettings []) + fromString str = MkCommand (toChType str) emptySettings emptyParameters -- ** SELECT @@ -52,10 +57,10 @@ -} data Select (columns :: [Type]) output where - MkSelect :: ([(Builder, Builder)] -> ChString) -> DbSettings -> Select columns output + MkSelect :: ([(Builder, Builder)] -> ChString) -> DbSettings -> QueryParameters -> Select columns output unsafeMkSelect :: ([(Builder, Builder)] -> Builder) -> Select columns output -unsafeMkSelect s = MkSelect (toChType . s) (MkDbSettings []) +unsafeMkSelect s = MkSelect (toChType . s) emptySettings emptyParameters {-| Type-safe wrapper for statements like @@ -114,10 +119,10 @@ -} data Insert (columns :: [Type]) output where - MkInsert :: ([(Builder, Builder)] -> ChString) -> DbSettings -> Insert columns output + MkInsert :: ([(Builder, Builder)] -> ChString) -> DbSettings -> QueryParameters -> Insert columns output unsafeMkInsert :: ([(Builder, Builder)] -> Builder) -> Insert columns output -unsafeMkInsert s = MkInsert (toChType . s) (MkDbSettings []) +unsafeMkInsert s = MkInsert (toChType . s) emptySettings emptyParameters intoTable :: forall name columns output @@ -171,4 +176,4 @@ parameter val = AddParameter (MkParamater val) renderParameter :: forall name chType . KnownParameter (Parameter name chType) => Parameter name chType -> Builder -renderParameter (MkParamater chType) = (byteString . BS8.pack . symbolVal @name) Proxy <> "=" <> toQueryPart chType +renderParameter (MkParamater chType) = (byteString . BS8.pack . symbolVal @name) Proxy <> "=" <> toQueryPartQuoted chType