dataframe-core 2.6.0.0 → 2.7.0.0
raw patch · 9 files changed
+633/−188 lines, 9 files
Files
- dataframe-core.cabal +2/−1
- src-internal/DataFrame/Internal/Aggregation/Plan.hs +26/−23
- src-internal/DataFrame/Internal/Expression.hs +146/−71
- src-internal/DataFrame/Internal/Expression/OpId.hs +260/−0
- src-internal/DataFrame/Internal/Expression/Operators.hs +117/−27
- src-internal/DataFrame/Internal/Expression/Simplify.hs +14/−10
- src-internal/DataFrame/Internal/Interpreter.hs +42/−40
- src-internal/DataFrame/Internal/Interpreter/Kernels.hs +16/−16
- src/DataFrame/Typed/Schema.hs +10/−0
dataframe-core.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe-core-version: 2.6.0.0+version: 2.7.0.0 synopsis: Core data structures for the dataframe library. description: Minimal interchange-format types for the @dataframe@ ecosystem:@@ -76,6 +76,7 @@ -- Expressions: syntax, operators, simplification. DataFrame.Internal.Expression DataFrame.Internal.Expression.Operators+ DataFrame.Internal.Expression.OpId DataFrame.Internal.Expression.Reflect DataFrame.Internal.Expression.Operators.Nullable DataFrame.Internal.Expression.Simplify
src-internal/DataFrame/Internal/Aggregation/Plan.hs view
@@ -2,7 +2,6 @@ {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE LambdaCase #-}-{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -28,10 +27,15 @@ getColumn, ) import DataFrame.Internal.Expression (+ AggId (..), AggStrategy (..),- BinaryOp (binaryCommutative, binaryName),+ BinOpId (..),+ BinaryOp (binaryId), Expr (..),+ OpRef (..), SomeExpr (..),+ UnOpId (..),+ UnaryOp (unaryId), ) import Type.Reflection (Typeable, typeRep) @@ -49,27 +53,25 @@ -- | Plan a named output expression, or 'Nothing' if the shape is unsupported. planAgg :: GroupedDataFrame -> SomeExpr -> Maybe AggPlan planAgg gdf (SomeExpr (expr :: Expr a)) = case expr of- Agg (FoldAgg tag _ _) (Col name) -> foldPlan tag name- Agg (MergeAgg tag _ _ _ _) (Col name) -> mergePlan tag name- Agg (CollectAgg tag _) (Col name) -> collectPlan tag name+ Agg (FoldAgg (Builtin tag) _ _) (Col name) -> foldPlan tag name+ Agg (MergeAgg (Builtin tag) _ _ _ _) (Col name) -> mergePlan tag name+ Agg (CollectAgg (Builtin tag) _) (Col name) -> collectPlan tag name Binary op- (Agg (FoldAgg lt Nothing _) (Col a))- (Agg (FoldAgg rt Nothing _) (Col b)) ->- if binaryName op == "sub" && lt == "maximum" && rt == "minimum"- then requireBoth a b (PlanMaxMinusMin a b)- else Nothing+ (Agg (FoldAgg (Builtin AMaximum) Nothing _) (Col a))+ (Agg (FoldAgg (Builtin AMinimum) Nothing _) (Col b))+ | Builtin BSub <- binaryId op -> requireBoth a b (PlanMaxMinusMin a b) _ -> Nothing where foldPlan tag name = case tag of- "sum" -> require name (PlanScatter RSum name)- "minimum" -> require name (PlanScatter RMin name)- "maximum" -> require name (PlanScatter RMax name)+ ASum -> require name (PlanScatter RSum name)+ AMinimum -> require name (PlanScatter RMin name)+ AMaximum -> require name (PlanScatter RMax name) _ -> Nothing mergePlan tag name = case tag of- "mean" -> outputType @Double >> require name (PlanScatter RMean name)- "count" -> outputType @Int >> require name (PlanScatter RCount name)- "countDistinct" -> do+ AMean -> outputType @Double >> require name (PlanScatter RMean name)+ ACount -> outputType @Int >> require name (PlanScatter RCount name)+ ACountDistinct -> do outputType @Int c <- getColumn name (fullDataframe gdf) if distinctSupported c then Just (PlanCountDistinct name) else Nothing@@ -79,9 +81,9 @@ Just Refl -> Just () Nothing -> Nothing collectPlan tag name = case tag of- "stddev" -> require name (PlanScatter RStd name)- "variance" -> require name (PlanScatter RVar name)- "median" -> require name (PlanMedian name)+ AStddev -> require name (PlanScatter RStd name)+ AVariance -> require name (PlanScatter RVar name)+ AMedian -> require name (PlanMedian name) _ -> Nothing require name plan = colUnboxedNumeric name >> Just plan requireBoth a b plan = colUnboxedNumeric a >> colUnboxedNumeric b >> Just plan@@ -162,8 +164,8 @@ -- | Tag a single named aggregation with its moment role, or reject the group. classify :: M.Map T.Text SomeExpr -> (T.Text, SomeExpr) -> Maybe (T.Text, Role) classify exprs (name, SomeExpr expr) = case expr of- Agg (MergeAgg "count" _ _ _ _) _ -> Just (name, RoleN)- Agg (FoldAgg "sum" _ _) arg -> (\t -> (name, termRole t)) <$> resolveTerm exprs (SomeExpr arg)+ Agg (MergeAgg (Builtin ACount) _ _ _ _) _ -> Just (name, RoleN)+ Agg (FoldAgg (Builtin ASum) _ _) arg -> (\t -> (name, termRole t)) <$> resolveTerm exprs (SomeExpr arg) _ -> Nothing termRole :: Term -> Role@@ -179,9 +181,10 @@ Col nm -> case M.lookup nm exprs of Just ue -> go (fuel - 1) ue Nothing -> Just (Lin nm)- Unary _ inner -> go (fuel - 1) (SomeExpr inner)+ Unary op inner+ | Builtin UToDouble <- unaryId op -> go (fuel - 1) (SomeExpr inner) Binary op l r- | binaryName op == "mult" && binaryCommutative op -> do+ | Builtin BMult <- binaryId op -> do Lin a <- go (fuel - 1) (SomeExpr l) Lin b <- go (fuel - 1) (SomeExpr r) Just (sortProd a b)
src-internal/DataFrame/Internal/Expression.hs view
@@ -13,7 +13,10 @@ {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE NoFieldSelectors #-} -module DataFrame.Internal.Expression where+module DataFrame.Internal.Expression (+ module DataFrame.Internal.Expression,+ module DataFrame.Internal.Expression.OpId,+) where import qualified Data.Map.Strict as M import Data.Maybe (fromMaybe)@@ -23,18 +26,19 @@ import qualified Data.Vector.Generic as VG import DataFrame.Internal.Column import qualified DataFrame.Internal.Display.Pretty as P+import DataFrame.Internal.Expression.OpId import Type.Reflection (Typeable, typeOf, typeRep) -- | Operators are an open typeclass so the simplifier can match them by 'cast'. class (Typeable op) => UnaryOp op where unaryFn :: op a b -> a -> b- unaryName :: op a b -> T.Text+ unaryId :: op a b -> OpRef UnOpId unarySymbol :: op a b -> Maybe T.Text unarySymbol _ = Nothing class (Typeable op) => BinaryOp op where binaryFn :: op a b c -> a -> b -> c- binaryName :: op a b c -> T.Text+ binaryId :: op a b c -> OpRef BinOpId binarySymbol :: op a b c -> Maybe T.Text binarySymbol _ = Nothing binaryCommutative :: op a b c -> Bool@@ -42,15 +46,23 @@ binaryPrecedence :: op a b c -> Int binaryPrecedence _ = 9 +-- | Display name of a unary operator.+unaryName :: (UnaryOp op) => op a b -> T.Text+unaryName = opRefName unOpName . unaryId++-- | Display name of a binary operator.+binaryName :: (BinaryOp op) => op a b c -> T.Text+binaryName = opRefName binOpName . binaryId+ data UnUDF a b = MkUnaryOp { unaryFn :: a -> b- , unaryName :: T.Text+ , unaryId :: OpRef UnOpId , unarySymbol :: Maybe T.Text } data BinUDF a b c = MkBinaryOp { binaryFn :: a -> b -> c- , binaryName :: T.Text+ , binaryId :: OpRef BinOpId , binarySymbol :: Maybe T.Text , binaryCommutative :: Bool , binaryPrecedence :: Int@@ -58,40 +70,50 @@ instance UnaryOp UnUDF where unaryFn (MkUnaryOp{unaryFn = f}) = f- unaryName (MkUnaryOp{unaryName = n}) = n+ unaryId (MkUnaryOp{unaryId = n}) = n unarySymbol (MkUnaryOp{unarySymbol = s}) = s instance BinaryOp BinUDF where binaryFn (MkBinaryOp{binaryFn = f}) = f- binaryName (MkBinaryOp{binaryName = n}) = n+ binaryId (MkBinaryOp{binaryId = n}) = n binarySymbol (MkBinaryOp{binarySymbol = s}) = s binaryCommutative (MkBinaryOp{binaryCommutative = c}) = c binaryPrecedence (MkBinaryOp{binaryPrecedence = p}) = p data AggStrategy a b where CollectAgg ::- (VG.Vector v b, Typeable v) => T.Text -> (v b -> a) -> AggStrategy a b- FoldAgg :: T.Text -> Maybe a -> (a -> b -> a) -> AggStrategy a b+ (VG.Vector v b, Typeable v) =>+ OpRef AggId -> (v b -> a) -> AggStrategy a b+ FoldAgg :: OpRef AggId -> Maybe a -> (a -> b -> a) -> AggStrategy a b MergeAgg :: (Columnable acc) =>- T.Text ->+ OpRef AggId -> acc -> (acc -> b -> acc) -> (acc -> acc -> acc) -> (acc -> a) -> AggStrategy a b +aggStrategyId :: AggStrategy a b -> OpRef AggId+aggStrategyId (CollectAgg i _) = i+aggStrategyId (FoldAgg i _ _) = i+aggStrategyId (MergeAgg i _ _ _ _) = i++-- | Display name of an aggregation.+aggStrategyName :: AggStrategy a b -> T.Text+aggStrategyName = opRefName aggName . aggStrategyId+ data Expr a where Col :: (Columnable a) => T.Text -> Expr a CastWith :: (Columnable a, Columnable b, Read a) => T.Text ->- T.Text ->+ OpRef CastId -> (Either String a -> b) -> Expr b CastExprWith :: (Columnable a, Columnable b, Columnable src, Read a) =>- T.Text ->+ OpRef CastId -> (Either String a -> b) -> Expr src -> Expr b@@ -131,7 +153,7 @@ Binary ( MkBinaryOp { binaryFn = (+)- , binaryName = "add"+ , binaryId = Builtin BAdd , binarySymbol = Just "+" , binaryCommutative = True , binaryPrecedence = 6@@ -143,7 +165,7 @@ Binary ( MkBinaryOp { binaryFn = (-)- , binaryName = "sub"+ , binaryId = Builtin BSub , binarySymbol = Just "-" , binaryCommutative = False , binaryPrecedence = 6@@ -155,7 +177,7 @@ Binary ( MkBinaryOp { binaryFn = (*)- , binaryName = "mult"+ , binaryId = Builtin BMult , binarySymbol = Just "*" , binaryCommutative = True , binaryPrecedence = 7@@ -168,15 +190,16 @@ negate :: Expr a -> Expr a negate = Unary- (MkUnaryOp{unaryFn = negate, unaryName = "negate", unarySymbol = Nothing})+ (MkUnaryOp{unaryFn = negate, unaryId = Builtin UNegate, unarySymbol = Nothing}) abs :: (Num a) => Expr a -> Expr a- abs = Unary (MkUnaryOp{unaryFn = abs, unaryName = "abs", unarySymbol = Nothing})+ abs =+ Unary (MkUnaryOp{unaryFn = abs, unaryId = Builtin UAbs, unarySymbol = Nothing}) signum :: (Num a) => Expr a -> Expr a signum = Unary- (MkUnaryOp{unaryFn = signum, unaryName = "signum", unarySymbol = Nothing})+ (MkUnaryOp{unaryFn = signum, unaryId = Builtin USignum, unarySymbol = Nothing}) add :: (Num a, Columnable a) => Expr a -> Expr a -> Expr a add = (+)@@ -196,7 +219,7 @@ Binary ( MkBinaryOp { binaryFn = (/)- , binaryName = "divide"+ , binaryId = Builtin BDivide , binarySymbol = Just "/" , binaryCommutative = False , binaryPrecedence = 7@@ -214,80 +237,96 @@ pi :: (Floating a, Columnable a) => Expr a pi = Lit pi exp :: (Floating a, Columnable a) => Expr a -> Expr a- exp = Unary (MkUnaryOp{unaryFn = exp, unaryName = "exp", unarySymbol = Nothing})+ exp =+ Unary (MkUnaryOp{unaryFn = exp, unaryId = Builtin UExp, unarySymbol = Nothing}) sqrt :: (Floating a, Columnable a) => Expr a -> Expr a sqrt =- Unary (MkUnaryOp{unaryFn = sqrt, unaryName = "sqrt", unarySymbol = Nothing})+ Unary+ (MkUnaryOp{unaryFn = sqrt, unaryId = Builtin USqrt, unarySymbol = Nothing}) (**) :: (Floating a, Columnable a) => Expr a -> Expr a -> Expr a (**) = Binary ( MkBinaryOp { binaryFn = (**)- , binaryName = "exponentiate"+ , binaryId = Builtin BExponentiate , binarySymbol = Just "**" , binaryCommutative = False , binaryPrecedence = 8 } ) log :: (Floating a, Columnable a) => Expr a -> Expr a- log = Unary (MkUnaryOp{unaryFn = log, unaryName = "log", unarySymbol = Nothing})+ log =+ Unary (MkUnaryOp{unaryFn = log, unaryId = Builtin ULog, unarySymbol = Nothing}) logBase :: (Floating a, Columnable a) => Expr a -> Expr a -> Expr a logBase = Binary ( MkBinaryOp { binaryFn = logBase- , binaryName = "logBase"+ , binaryId = Builtin BLogBase , binarySymbol = Nothing , binaryCommutative = False , binaryPrecedence = 1 } ) sin :: (Floating a, Columnable a) => Expr a -> Expr a- sin = Unary (MkUnaryOp{unaryFn = sin, unaryName = "sin", unarySymbol = Nothing})+ sin =+ Unary (MkUnaryOp{unaryFn = sin, unaryId = Builtin USin, unarySymbol = Nothing}) cos :: (Floating a, Columnable a) => Expr a -> Expr a- cos = Unary (MkUnaryOp{unaryFn = cos, unaryName = "cos", unarySymbol = Nothing})+ cos =+ Unary (MkUnaryOp{unaryFn = cos, unaryId = Builtin UCos, unarySymbol = Nothing}) tan :: (Floating a, Columnable a) => Expr a -> Expr a- tan = Unary (MkUnaryOp{unaryFn = tan, unaryName = "tan", unarySymbol = Nothing})+ tan =+ Unary (MkUnaryOp{unaryFn = tan, unaryId = Builtin UTan, unarySymbol = Nothing}) asin :: (Floating a, Columnable a) => Expr a -> Expr a asin =- Unary (MkUnaryOp{unaryFn = asin, unaryName = "asin", unarySymbol = Nothing})+ Unary+ (MkUnaryOp{unaryFn = asin, unaryId = Builtin UAsin, unarySymbol = Nothing}) acos :: (Floating a, Columnable a) => Expr a -> Expr a acos =- Unary (MkUnaryOp{unaryFn = acos, unaryName = "acos", unarySymbol = Nothing})+ Unary+ (MkUnaryOp{unaryFn = acos, unaryId = Builtin UAcos, unarySymbol = Nothing}) atan :: (Floating a, Columnable a) => Expr a -> Expr a atan =- Unary (MkUnaryOp{unaryFn = atan, unaryName = "atan", unarySymbol = Nothing})+ Unary+ (MkUnaryOp{unaryFn = atan, unaryId = Builtin UAtan, unarySymbol = Nothing}) sinh :: (Floating a, Columnable a) => Expr a -> Expr a sinh =- Unary (MkUnaryOp{unaryFn = sinh, unaryName = "sinh", unarySymbol = Nothing})+ Unary+ (MkUnaryOp{unaryFn = sinh, unaryId = Builtin USinh, unarySymbol = Nothing}) cosh :: (Floating a, Columnable a) => Expr a -> Expr a cosh =- Unary (MkUnaryOp{unaryFn = cosh, unaryName = "cosh", unarySymbol = Nothing})+ Unary+ (MkUnaryOp{unaryFn = cosh, unaryId = Builtin UCosh, unarySymbol = Nothing}) asinh :: (Floating a, Columnable a) => Expr a -> Expr a asinh = Unary- (MkUnaryOp{unaryFn = asinh, unaryName = "asinh", unarySymbol = Nothing})+ (MkUnaryOp{unaryFn = asinh, unaryId = Builtin UAsinh, unarySymbol = Nothing}) acosh :: (Floating a, Columnable a) => Expr a -> Expr a acosh = Unary- (MkUnaryOp{unaryFn = acosh, unaryName = "acosh", unarySymbol = Nothing})+ (MkUnaryOp{unaryFn = acosh, unaryId = Builtin UAcosh, unarySymbol = Nothing}) atanh :: (Floating a, Columnable a) => Expr a -> Expr a atanh = Unary- (MkUnaryOp{unaryFn = atanh, unaryName = "atanh", unarySymbol = Nothing})+ (MkUnaryOp{unaryFn = atanh, unaryId = Builtin UAtanh, unarySymbol = Nothing}) instance (Show a) => Show (Expr a) where show :: Expr a -> String show (Col name) = "(col @" ++ show (typeRep @a) ++ " " ++ show name ++ ")"- show (CastWith name tag _) = "(castWith " ++ show tag ++ " " ++ show name ++ ")"- show (CastExprWith tag _ inner) = "(castExprWith " ++ show tag ++ " " ++ show inner ++ ")"+ show (CastWith name tag _) =+ "(castWith " ++ show (opRefName castName tag) ++ " " ++ show name ++ ")"+ show (CastExprWith tag _ inner) =+ "(castExprWith "+ ++ show (opRefName castExprName tag)+ ++ " "+ ++ show inner+ ++ ")" show (Lit value) = "(lit (" ++ show value ++ "))" show (If cond l r) = "(ifThenElse " ++ show cond ++ " " ++ show l ++ " " ++ show r ++ ")" show (Unary op value) = "(" ++ T.unpack (unaryName op) ++ " " ++ show value ++ ")" show (Binary op a b) = "(" ++ T.unpack (binaryName op) ++ " " ++ show a ++ " " ++ show b ++ ")"- show (Agg (CollectAgg op _) expr) = "(" ++ T.unpack op ++ " " ++ show expr ++ ")"- show (Agg (FoldAgg op _ _) expr) = "(" ++ T.unpack op ++ " " ++ show expr ++ ")"- show (Agg (MergeAgg op _ _ _ _) expr) = "(" ++ T.unpack op ++ " " ++ show expr ++ ")"+ show (Agg strat expr) =+ "(" ++ T.unpack (aggStrategyName strat) ++ " " ++ show expr ++ ")" show (Over keys inner) = "(over " ++ show keys ++ " " ++ show inner ++ ")" normalize :: (Show a, Typeable a) => Expr a -> Expr a@@ -318,41 +357,77 @@ where exprKey :: Expr a -> String exprKey (Col name) = "0:" ++ T.unpack name- exprKey (CastWith name tag _) = "0CW:" ++ T.unpack name ++ ":" ++ T.unpack tag- exprKey (CastExprWith tag _ _) = "0CE:" ++ T.unpack tag+ exprKey (CastWith name tag _) = "0CW:" ++ T.unpack name ++ ":" ++ idKey tag+ exprKey (CastExprWith tag _ _) = "0CE:" ++ idKey tag exprKey (Lit val) = "1:" ++ show val exprKey (If c t e) = "2:" ++ exprKey c ++ exprKey t ++ exprKey e- exprKey (Unary op e) = "3:" ++ T.unpack (unaryName op) ++ exprKey e- exprKey (Binary op e1' e2') = "4:" ++ T.unpack (binaryName op) ++ exprKey e1' ++ exprKey e2'- exprKey (Agg (CollectAgg name _) e) = "5:" ++ T.unpack name ++ exprKey e- exprKey (Agg (FoldAgg name _ _) e) = "5:" ++ T.unpack name ++ exprKey e- exprKey (Agg (MergeAgg name _ _ _ _) e) = "5:" ++ T.unpack name ++ exprKey e+ exprKey (Unary op e) = "3:" ++ idKey (unaryId op) ++ exprKey e+ exprKey (Binary op e1' e2') =+ "4:" ++ idKey (binaryId op) ++ exprKey e1' ++ exprKey e2'+ exprKey (Agg strat e) = "5:" ++ idKey (aggStrategyId strat) ++ exprKey e exprKey (Over keys e) = "6:over:" ++ show keys ++ exprKey e+ idKey :: (Show k) => OpRef k -> String+ idKey (Builtin k) = "b:" ++ show k+ idKey (Named n) = "n:" ++ T.unpack n+ idKey (Anonymous l) = "a:" ++ T.unpack l +-- | A key equal only for provably equal expressions; Nothing if anonymous.+structuralKey :: (Columnable a) => Expr a -> Maybe String+structuralKey = go . normalize+ where+ go :: forall x. Expr x -> Maybe String+ go e = case e of+ Col name -> Just ("c" ++ show name ++ "@" ++ show (typeRep @x))+ CastWith name tag _ ->+ idKey tag >>= \k -> Just ("w" ++ show name ++ k ++ "@" ++ show (typeRep @x))+ CastExprWith tag _ inner -> idKey tag >>= \k -> wrap ("x" ++ k) [go inner]+ Lit v -> Just ("l" ++ show v ++ "@" ++ show (typeRep @x))+ If c t f -> wrap "if" [go c, go t, go f]+ Unary op inner -> idKey (unaryId op) >>= \k -> wrap k [go inner]+ Binary op l r -> idKey (binaryId op) >>= \k -> wrap k [go l, go r]+ Agg strat inner ->+ idKey (aggStrategyId strat) >>= \k -> wrap ("agg" ++ k) [go inner]+ Over keys inner -> wrap ("over" ++ show keys) [go inner]+ wrap tag parts = (\ps -> tag ++ "(" ++ concatMap (++ ",") ps ++ ")") <$> sequence parts+ idKey :: (Show k) => OpRef k -> Maybe String+ idKey (Builtin k) = Just ("b" ++ show k)+ idKey (Named n) = Just ("n" ++ show n)+ idKey (Anonymous _) = Nothing++-- | Provable equality; anonymous operators are never equal. eqExpr :: forall a. (Columnable a) => Expr a -> Expr a -> Bool eqExpr l r = eqNormalized (normalize l) (normalize r)++eqNormalized :: forall a. Expr a -> Expr a -> Bool+eqNormalized = go where- exprEq :: (Columnable b, Columnable c) => Expr b -> Expr c -> Bool- exprEq e1 e2 = case testEquality (typeOf e1) (typeOf e2) of- Just Refl -> eqExpr e1 e2+ sub :: forall b c. (Columnable b, Columnable c) => Expr b -> Expr c -> Bool+ sub e1 e2 = case testEquality (typeRep @b) (typeRep @c) of+ Just Refl -> go e1 e2 Nothing -> False- eqNormalized :: Expr a -> Expr a -> Bool- eqNormalized (Col n1) (Col n2) = n1 == n2- eqNormalized (CastWith n1 t1 _) (CastWith n2 t2 _) = n1 == n2 && t1 == t2- eqNormalized (CastExprWith t1 _ e1) (CastExprWith t2 _ e2) = t1 == t2 && e1 `exprEq` e2- eqNormalized (Lit v1) (Lit v2) = v1 == v2- eqNormalized (If c1 t1 e1) (If c2 t2 e2) =- eqExpr c1 c2 && t1 `exprEq` t2 && e1 `exprEq` e2- eqNormalized (Unary op1 e1) (Unary op2 e2) = unaryName op1 == unaryName op2 && e1 `exprEq` e2- eqNormalized (Binary op1 e1a e1b) (Binary op2 e2a e2b) = binaryName op1 == binaryName op2 && e1a `exprEq` e2a && e1b `exprEq` e2b- eqNormalized (Agg (CollectAgg n1 _) e1) (Agg (CollectAgg n2 _) e2) =- n1 == n2 && e1 `exprEq` e2- eqNormalized (Agg (FoldAgg n1 _ _) e1) (Agg (FoldAgg n2 _ _) e2) =- n1 == n2 && e1 `exprEq` e2- eqNormalized (Agg (MergeAgg n1 _ _ _ _) e1) (Agg (MergeAgg n2 _ _ _ _) e2) =- n1 == n2 && e1 `exprEq` e2- eqNormalized (Over k1 e1) (Over k2 e2) = k1 == k2 && e1 `exprEq` e2- eqNormalized _ _ = False+ go :: forall x. Expr x -> Expr x -> Bool+ go (Col n1) (Col n2) = n1 == n2+ go (CastWith n1 t1 _) (CastWith n2 t2 _) = n1 == n2 && sameOp t1 t2+ go (CastExprWith t1 _ e1) (CastExprWith t2 _ e2) = sameOp t1 t2 && e1 `sub` e2+ go (Lit v1) (Lit v2) = v1 == v2+ go (If c1 t1 e1) (If c2 t2 e2) = go c1 c2 && go t1 t2 && go e1 e2+ go (Unary op1 e1) (Unary op2 e2) =+ sameOp (unaryId op1) (unaryId op2) && e1 `sub` e2+ go (Binary op1 e1a e1b) (Binary op2 e2a e2b) =+ sameOp (binaryId op1) (binaryId op2)+ && e1a `sub` e2a+ && e1b `sub` e2b+ go (Agg s1 e1) (Agg s2 e2) =+ sameStrategy s1 s2+ && sameOp (aggStrategyId s1) (aggStrategyId s2)+ && e1 `sub` e2+ go (Over k1 e1) (Over k2 e2) = k1 == k2 && go e1 e2+ go _ _ = False+ sameStrategy :: AggStrategy p q -> AggStrategy p' q' -> Bool+ sameStrategy CollectAgg{} CollectAgg{} = True+ sameStrategy FoldAgg{} FoldAgg{} = True+ sameStrategy MergeAgg{} MergeAgg{} = True+ sameStrategy _ _ = False replaceExpr :: forall a b.@@ -435,7 +510,8 @@ toDoc prec expr = case expr of Col name -> P.text (T.unpack name) CastWith name _ _ -> P.text (T.unpack name)- CastExprWith tag _ inner -> P.text (T.unpack tag) <> P.parens (toDoc 0 inner)+ CastExprWith tag _ inner ->+ P.text (T.unpack (opRefName castExprName tag)) <> P.parens (toDoc 0 inner) Lit value -> P.text (show value) If{} -> renderIf prec expr Unary op arg ->@@ -446,9 +522,8 @@ Nothing -> P.text (T.unpack (binaryName op)) <> P.parens (toDoc 0 l <> P.text ", " <> toDoc 0 r)- Agg (CollectAgg op _) arg -> P.text (T.unpack op) <> P.parens (toDoc 0 arg)- Agg (FoldAgg op _ _) arg -> P.text (T.unpack op) <> P.parens (toDoc 0 arg)- Agg (MergeAgg op _ _ _ _) arg -> P.text (T.unpack op) <> P.parens (toDoc 0 arg)+ Agg strat arg ->+ P.text (T.unpack (aggStrategyName strat)) <> P.parens (toDoc 0 arg) Over keys inner -> toDoc 0 inner <> P.text (".over(" ++ show (map T.unpack keys) ++ ")")
+ src-internal/DataFrame/Internal/Expression/OpId.hs view
@@ -0,0 +1,260 @@+{-# LANGUAGE DeriveFunctor #-}+{-# LANGUAGE OverloadedStrings #-}++-- | What an operator computes; kernels key on these tags, not display names.+module DataFrame.Internal.Expression.OpId (+ OpRef (..),+ UnOpId (..),+ BinOpId (..),+ AggId (..),+ CastId (..),+ sameOp,+ builtinOf,+ opRefName,+ unOpName,+ binOpName,+ aggName,+ castName,+ castExprName,+) where++import qualified Data.Text as T++-- | Builtin: library op; Named: equal names, equal functions; Anonymous: label.+data OpRef k+ = Builtin !k+ | Named !T.Text+ | Anonymous !T.Text+ deriving (Show, Functor)++-- | Provable equality: 'Anonymous' is never equal to anything, even itself.+sameOp :: (Eq k) => OpRef k -> OpRef k -> Bool+sameOp (Builtin a) (Builtin b) = a == b+sameOp (Named a) (Named b) = a == b+sameOp _ _ = False+{-# INLINE sameOp #-}++-- | The builtin tag, if any; the only thing kernels may dispatch on.+builtinOf :: OpRef k -> Maybe k+builtinOf (Builtin k) = Just k+builtinOf _ = Nothing+{-# INLINE builtinOf #-}++-- | The display name: the builtin's name, the given name, or the label.+opRefName :: (k -> T.Text) -> OpRef k -> T.Text+opRefName f (Builtin k) = f k+opRefName _ (Named n) = n+opRefName _ (Anonymous l) = l++-- | Library unary operators. Parameters are part of the identity.+data UnOpId+ = UNegate+ | UAbs+ | USignum+ | UExp+ | USqrt+ | ULog+ | USin+ | UCos+ | UTan+ | UAsin+ | UAcos+ | UAtan+ | USinh+ | UCosh+ | UAsinh+ | UAcosh+ | UAtanh+ | UNot+ | UToDouble+ | URelu+ | UToMaybe+ | -- | The default, rendered with 'show'.+ UFromMaybe !T.Text+ | UIsJust+ | UIsNothing+ | UFromJust+ | UFirstOrNothing+ | ULastOrNothing+ | UTextLength+ | USplitOn !T.Text+ | UMatch !T.Text+ | UMatchAll !T.Text+ | UParseDate !T.Text+ | -- | The mapping, rendered with 'show'.+ URecode !T.Text+ | -- | The default and the mapping, rendered with 'show'.+ URecodeWithDefault !T.Text !T.Text+ deriving (Eq, Ord, Show)++-- | Library binary operators.+data BinOpId+ = BAdd+ | BSub+ | BMult+ | BDivide+ | BExponentiate+ | BLogBase+ | BDiv+ | BMod+ | -- | @(^)@: integral exponent.+ BPow+ | -- | @(^^)@: fractional base, integral exponent.+ BPowFrac+ | BEq+ | BNeq+ | BLt+ | BLeq+ | BGt+ | BGeq+ | BAnd+ | BOr+ | BNullAnd+ | BNullOr+ | BNullAdd+ | BNullSub+ | BNullMul+ | BNullDiv+ | BMin+ | BMax+ | BContains+ | BDaysBetween+ deriving (Eq, Ord, Show, Enum, Bounded)++-- | Library aggregations.+data AggId+ = ASum+ | AMinimum+ | AMaximum+ | AMean+ | ACount+ | ACountDistinct+ | AMode+ | ACollect+ | ASumMaybe+ | AMeanMaybe+ | AMedianMaybe+ | AVariance+ | AMedian+ | AStddev+ | AStddevMaybe+ | APercentile !Int+ | ATopK !Int+ | ABottomK !Int+ deriving (Eq, Ord, Show)++-- | Display and wire name of a unary builtin.+unOpName :: UnOpId -> T.Text+unOpName op = case op of+ UNegate -> "negate"+ UAbs -> "abs"+ USignum -> "signum"+ UExp -> "exp"+ USqrt -> "sqrt"+ ULog -> "log"+ USin -> "sin"+ UCos -> "cos"+ UTan -> "tan"+ UAsin -> "asin"+ UAcos -> "acos"+ UAtan -> "atan"+ USinh -> "sinh"+ UCosh -> "cosh"+ UAsinh -> "asinh"+ UAcosh -> "acosh"+ UAtanh -> "atanh"+ UNot -> "not"+ UToDouble -> "toDouble"+ URelu -> "relu"+ UToMaybe -> "toMaybe"+ UFromMaybe _ -> "fromMaybe"+ UIsJust -> "isJust"+ UIsNothing -> "isNothing"+ UFromJust -> "fromJust"+ UFirstOrNothing -> "firstOrNothing"+ ULastOrNothing -> "lastOrNothing"+ UTextLength -> "textLength"+ USplitOn _ -> "splitOn"+ UMatch r -> "match " <> T.pack (show r)+ UMatchAll r -> "matchAll " <> T.pack (show r)+ UParseDate f -> "parseDate " <> f+ URecode m -> "recode " <> m+ URecodeWithDefault d m -> "recodeWithDefault " <> d <> " " <> m++-- | Display and wire name of a binary builtin.+binOpName :: BinOpId -> T.Text+binOpName op = case op of+ BAdd -> "add"+ BSub -> "sub"+ BMult -> "mult"+ BDivide -> "divide"+ BExponentiate -> "exponentiate"+ BLogBase -> "logBase"+ BDiv -> "div"+ BMod -> "mod"+ BPow -> "pow"+ BPowFrac -> "pow"+ BEq -> "eq"+ BNeq -> "neq"+ BLt -> "lt"+ BLeq -> "leq"+ BGt -> "gt"+ BGeq -> "geq"+ BAnd -> "and"+ BOr -> "or"+ BNullAnd -> "nulland"+ BNullOr -> "nullor"+ BNullAdd -> "nulladd"+ BNullSub -> "nullsub"+ BNullMul -> "nullmul"+ BNullDiv -> "nulldiv"+ BMin -> "min"+ BMax -> "max"+ BContains -> "contains"+ BDaysBetween -> "daysBetween"++-- | Display and wire name of an aggregation builtin.+aggName :: AggId -> T.Text+aggName op = case op of+ ASum -> "sum"+ AMinimum -> "minimum"+ AMaximum -> "maximum"+ AMean -> "mean"+ ACount -> "count"+ ACountDistinct -> "countDistinct"+ AMode -> "mode"+ ACollect -> "collect"+ ASumMaybe -> "sumMaybe"+ AMeanMaybe -> "meanMaybe"+ AMedianMaybe -> "medianMaybe"+ AVariance -> "variance"+ AMedian -> "median"+ AStddev -> "stddev"+ AStddevMaybe -> "stddevMaybe"+ APercentile n -> "percentile " <> T.pack (show n)+ ATopK k -> "topK_" <> T.pack (show k)+ ABottomK k -> "bottomK_" <> T.pack (show k)++-- | Library casts; a default value is shown so different defaults differ.+data CastId+ = CCast+ | CCastWithDefault !T.Text+ | CCastEither+ | CUnsafeCast+ deriving (Eq, Ord, Show)++-- | Display name of a cast reading a column by name.+castName :: CastId -> T.Text+castName c = case c of+ CCast -> "cast"+ CCastWithDefault d -> "castWithDefault:" <> d+ CCastEither -> "castEither"+ CUnsafeCast -> "unsafeCast"++-- | Display name of a cast applied to an expression.+castExprName :: CastId -> T.Text+castExprName c = case c of+ CCast -> "castExpr"+ CCastWithDefault d -> "castExprWithDefault:" <> d+ CCastEither -> "castExprEither"+ CUnsafeCast -> "unsafeCastExpr"
src-internal/DataFrame/Internal/Expression/Operators.hs view
@@ -7,21 +7,25 @@ module DataFrame.Internal.Expression.Operators where import Data.Function ((&))+import Data.Int (Int32, Int64) import qualified Data.Text as T import DataFrame.Internal.Column (Columnable) import DataFrame.Internal.Column.Types (Promote, PromoteDiv) import DataFrame.Internal.Expression (+ BinOpId (..), BinUDF (MkBinaryOp), BinaryOp ( binaryCommutative, binaryFn,- binaryName,+ binaryId, binaryPrecedence, binarySymbol ), Expr (Binary, Col, If, Lit, Unary), NamedExpr,+ OpRef (..), SomeExpr (SomeExpr),+ UnOpId, UnUDF (MkUnaryOp), ) import DataFrame.Internal.Expression.Operators.Nullable (@@ -61,6 +65,74 @@ col :: (Columnable a) => T.Text -> Expr a col = Col +-- | Column reference of type Int; same as `col @Int`.+int :: T.Text -> Expr Int+int = Col++-- | Column reference of type Integer; same as `col @Integer`.+integer :: T.Text -> Expr Integer+integer = Col++-- | Column reference of type Double; same as `col @Double`.+double :: T.Text -> Expr Double+double = Col++-- | Column reference of type Text; same as `col @T.Text`.+text :: T.Text -> Expr T.Text+text = Col++-- | Column reference of type Bool; same as `col @Bool`.+bool :: T.Text -> Expr Bool+bool = Col++-- | Nullable column reference of type Maybe Int.+maybeInt :: T.Text -> Expr (Maybe Int)+maybeInt = Col++-- | Nullable column reference of type Maybe Double.+maybeDouble :: T.Text -> Expr (Maybe Double)+maybeDouble = Col++-- | Nullable column reference of type Maybe Text.+maybeText :: T.Text -> Expr (Maybe T.Text)+maybeText = Col++-- | Nullable column reference of type Maybe Bool.+maybeBool :: T.Text -> Expr (Maybe Bool)+maybeBool = Col++-- | Column reference of type Int32.+int32 :: T.Text -> Expr Int32+int32 = Col++-- | Column reference of type Int64.+int64 :: T.Text -> Expr Int64+int64 = Col++-- | Column reference of type Float (32-bit).+float32 :: T.Text -> Expr Float+float32 = Col++-- | Column reference of type Double (64-bit).+float64 :: T.Text -> Expr Double+float64 = Col++-- | Nullable column reference of type Maybe Int32.+maybeInt32 :: T.Text -> Expr (Maybe Int32)+maybeInt32 = Col++-- | Nullable column reference of type Maybe Int64.+maybeInt64 :: T.Text -> Expr (Maybe Int64)+maybeInt64 = Col++-- | Nullable column reference of type Maybe Float (32-bit).+maybeFloat32 :: T.Text -> Expr (Maybe Float)+maybeFloat32 = Col++-- | Nullable column reference of type Maybe Double (64-bit).+maybeFloat64 :: T.Text -> Expr (Maybe Double)+maybeFloat64 = Col+ ifThenElse :: (Columnable a) => Expr Bool -> Expr a -> Expr a -> Expr a ifThenElse = If @@ -77,11 +149,17 @@ (.=) :: (Columnable a) => T.Text -> Expr a -> NamedExpr (.=) = flip as +-- | A user op; equal names must mean equal functions. Never hits builtins. liftDecorated :: (Columnable a, Columnable b) => (a -> b) -> T.Text -> Maybe T.Text -> Expr a -> Expr b-liftDecorated f opName rep = Unary (MkUnaryOp f opName rep)+liftDecorated f opName rep = Unary (MkUnaryOp f (Named opName) rep) +liftBuiltin ::+ (Columnable a, Columnable b) =>+ UnOpId -> (a -> b) -> Maybe T.Text -> Expr a -> Expr b+liftBuiltin op f rep = Unary (MkUnaryOp f (Builtin op) rep)+ lift2Decorated :: (Columnable c, Columnable b, Columnable a) => (c -> b -> a) ->@@ -93,8 +171,20 @@ Expr b -> Expr a lift2Decorated f opName rep comm prec =- Binary (MkBinaryOp f opName rep comm prec)+ Binary (MkBinaryOp f (Named opName) rep comm prec) +lift2Builtin ::+ (Columnable c, Columnable b, Columnable a) =>+ BinOpId ->+ (c -> b -> a) ->+ Maybe T.Text ->+ Bool ->+ Int ->+ Expr c ->+ Expr b ->+ Expr a+lift2Builtin op f rep comm prec = Binary (MkBinaryOp f (Builtin op) rep comm prec)+ data NullEq a b c where NullEq :: ( NumericWidenOp (BaseType a) (BaseType b)@@ -155,45 +245,45 @@ instance BinaryOp NullEq where binaryFn NullEq = applyNull2 (widenCmpOp (==))- binaryName NullEq = "eq"+ binaryId NullEq = Builtin BEq binarySymbol NullEq = Just ".==" binaryCommutative NullEq = True binaryPrecedence NullEq = 4 instance BinaryOp NullNeq where binaryFn NullNeq = applyNull2 (widenCmpOp (/=))- binaryName NullNeq = "neq"+ binaryId NullNeq = Builtin BNeq binarySymbol NullNeq = Just "./=" binaryCommutative NullNeq = True binaryPrecedence NullNeq = 4 instance BinaryOp NullLt where binaryFn NullLt = applyNull2 (widenCmpOp (<))- binaryName NullLt = "lt"+ binaryId NullLt = Builtin BLt binarySymbol NullLt = Just ".<" binaryPrecedence NullLt = 4 instance BinaryOp NullGt where binaryFn NullGt = applyNull2 (widenCmpOp (>))- binaryName NullGt = "gt"+ binaryId NullGt = Builtin BGt binarySymbol NullGt = Just ".>" binaryPrecedence NullGt = 4 instance BinaryOp NullLeq where binaryFn NullLeq = applyNull2 (widenCmpOp (<=))- binaryName NullLeq = "leq"+ binaryId NullLeq = Builtin BLeq binarySymbol NullLeq = Just ".<=" binaryPrecedence NullLeq = 4 instance BinaryOp NullGeq where binaryFn NullGeq = applyNull2 (widenCmpOp (>=))- binaryName NullGeq = "geq"+ binaryId NullGeq = Builtin BGeq binarySymbol NullGeq = Just ".>=" binaryPrecedence NullGeq = 4 instance BinaryOp NullAnd where binaryFn NullAnd = nullCmpOp (&&)- binaryName NullAnd = "nulland"+ binaryId NullAnd = Builtin BNullAnd binarySymbol NullAnd = Just ".&&" binaryCommutative NullAnd = True binaryPrecedence NullAnd = 3 instance BinaryOp NullOr where binaryFn NullOr = nullCmpOp (||)- binaryName NullOr = "nullor"+ binaryId NullOr = Builtin BNullOr binarySymbol NullOr = Just ".||" binaryCommutative NullOr = True binaryPrecedence NullOr = 2@@ -203,42 +293,42 @@ Expr a -> Expr a -> Expr Bool-(.==.) = lift2Decorated (==) "eq" (Just ".==.") True 4+(.==.) = lift2Builtin BEq (==) (Just ".==.") True 4 (./=.) :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool-(./=.) = lift2Decorated (/=) "neq" (Just "./=.") True 4+(./=.) = lift2Builtin BNeq (/=) (Just "./=.") True 4 (.<.) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(.<.) = lift2Decorated (<) "lt" (Just ".<.") False 4+(.<.) = lift2Builtin BLt (<) (Just ".<.") False 4 (.>.) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(.>.) = lift2Decorated (>) "gt" (Just ".>.") False 4+(.>.) = lift2Builtin BGt (>) (Just ".>.") False 4 (.<=.) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(.<=.) = lift2Decorated (<=) "leq" (Just ".<=.") False 4+(.<=.) = lift2Builtin BLeq (<=) (Just ".<=.") False 4 (.>=.) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(.>=.) = lift2Decorated (>=) "geq" (Just ".>=.") False 4+(.>=.) = lift2Builtin BGeq (>=) (Just ".>=.") False 4 (.+.) :: (Columnable a, Num a) => Expr a -> Expr a -> Expr a (.+.) = (+)@@ -263,7 +353,7 @@ Expr a -> Expr b -> Expr (WidenResult a b)-(.+) = lift2Decorated (applyNull2 (widenArithOp (+))) "nulladd" (Just ".+") True 6+(.+) = lift2Builtin BNullAdd (applyNull2 (widenArithOp (+))) (Just ".+") True 6 -- | Nullable-aware subtraction. (.-) ::@@ -274,7 +364,7 @@ Expr a -> Expr b -> Expr (WidenResult a b)-(.-) = lift2Decorated (applyNull2 (widenArithOp (-))) "nullsub" (Just ".-") False 6+(.-) = lift2Builtin BNullSub (applyNull2 (widenArithOp (-))) (Just ".-") False 6 -- | Nullable-aware multiplication. (.*) ::@@ -285,7 +375,7 @@ Expr a -> Expr b -> Expr (WidenResult a b)-(.*) = lift2Decorated (applyNull2 (widenArithOp (*))) "nullmul" (Just ".*") True 7+(.*) = lift2Builtin BNullMul (applyNull2 (widenArithOp (*))) (Just ".*") True 7 -- | Nullable-aware division. Integral operands are promoted to Double. (./) ::@@ -296,7 +386,7 @@ Expr a -> Expr b -> Expr (WidenResultDiv a b)-(./) = lift2Decorated (applyNull2 (divArithOp (/))) "nulldiv" (Just "./") False 7+(./) = lift2Builtin BNullDiv (applyNull2 (divArithOp (/))) (Just "./") False 7 -- Three-valued comparisons: Nothing if either operand is Nothing. @@ -367,10 +457,10 @@ (.>=) = Binary NullGeq (.&&.) :: Expr Bool -> Expr Bool -> Expr Bool-(.&&.) = lift2Decorated (&&) "and" (Just ".&&.") True 3+(.&&.) = lift2Builtin BAnd (&&) (Just ".&&.") True 3 (.||.) :: Expr Bool -> Expr Bool -> Expr Bool-(.||.) = lift2Decorated (||) "or" (Just ".||.") True 2+(.||.) = lift2Builtin BOr (||) (Just ".||.") True 2 -- | Nullable-aware logical AND, giving @Maybe Bool@ if either side is nullable. (.&&) ::@@ -398,7 +488,7 @@ , Num (Promote (BaseType a) (BaseType b)) ) => Expr a -> Expr b -> Expr a-(.^^) = lift2Decorated (applyNull2 (^^)) "pow" (Just ".^^") False 8+(.^^) = lift2Builtin BPowFrac (applyNull2 (^^)) (Just ".^^") False 8 (.^) :: ( Columnable (BaseType a)@@ -410,16 +500,16 @@ , Num (Promote (BaseType a) (BaseType b)) ) => Expr a -> Expr b -> Expr a-(.^) = lift2Decorated (applyNull2 (^)) "pow" (Just ".^") False 8+(.^) = lift2Builtin BPow (applyNull2 (^)) (Just ".^") False 8 -- Same-type (non-nullable) exponentiation operators (.^^.) :: (Columnable a, Columnable b, Fractional a, Integral b) => Expr a -> Expr b -> Expr a-(.^^.) = lift2Decorated (^^) "pow" (Just ".^^.") False 8+(.^^.) = lift2Builtin BPowFrac (^^) (Just ".^^.") False 8 (.^.) :: (Columnable a, Columnable b, Num a, Integral b) => Expr a -> Expr b -> Expr a-(.^.) = lift2Decorated (^) "pow" (Just ".^.") False 8+(.^.) = lift2Builtin BPow (^) (Just ".^.") False 8
src-internal/DataFrame/Internal/Expression/Simplify.hs view
@@ -27,9 +27,11 @@ import DataFrame.Internal.Expression ( BinaryOp, Expr (..),- UnaryOp (unaryName),+ OpRef (..),+ UnOpId (..),+ UnaryOp (unaryId), eqExpr,- normalize,+ structuralKey, ) import DataFrame.Internal.Expression.Operators ( NullAnd,@@ -80,7 +82,7 @@ Unary (op :: op b a) inner | Just Refl <- testEquality (typeRep @a) (typeRep @Bool) , Just Refl <- testEquality (typeRep @b) (typeRep @Bool)- , unaryName op == "not" ->+ , Builtin UNot <- unaryId op -> simplifyNot op (simplifyB inner) | otherwise -> expr If c t f ->@@ -104,7 +106,7 @@ Just b -> Lit (not b) Nothing -> case inner of Unary (op2 :: op2 b2 Bool) inner2- | unaryName op2 == "not"+ | Builtin UNot <- unaryId op2 , Just Refl <- testEquality (typeRep @b2) (typeRep @Bool) -> inner2 _ -> Unary op inner@@ -214,7 +216,7 @@ -- | True for a column lifted from an integral type (never NaN). integralColE :: forall c. (Columnable c) => Expr c -> Bool-integralColE (Unary op _) = unaryName op == "toDouble"+integralColE (Unary op _) | Builtin UToDouble <- unaryId op = True integralColE _ = or [ matches @Int@@ -226,15 +228,17 @@ atomOf :: forall a. (Columnable a) => Expr a -> Maybe Atom atomOf (Unary fm (Binary (op :: op c b r) (colE :: Expr c) litE))- | unaryName fm == "fromMaybe"+ | Builtin (UFromMaybe "False") <- unaryId fm , Just cmp <- cmpOf op- , Just t <- litDouble litE =- Just (Atom cmp t (show (normalize colE)) FalseOnNull (integralColE colE))+ , Just t <- litDouble litE+ , Just key <- structuralKey colE =+ Just (Atom cmp t key FalseOnNull (integralColE colE)) atomOf (Binary (op :: op c b a) (colE :: Expr c) litE) | Just cmp <- cmpOf op- , Just t <- litDouble litE =+ , Just t <- litDouble litE+ , Just key <- structuralKey colE = let nk = if isMaybeTy @c then UnknownOnNull else Total- in Just (Atom cmp t (show (normalize colE)) nk (integralColE colE))+ in Just (Atom cmp t key nk (integralColE colE)) atomOf _ = Nothing simplifyPredicatePair ::
src-internal/DataFrame/Internal/Interpreter.hs view
@@ -383,7 +383,7 @@ builtinBinary :: forall c b a. (Columnable c, Columnable b, Columnable a) =>- T.Text -> Maybe (Value c -> Value b -> Maybe (Value a))+ BinOpId -> Maybe (Value c -> Value b -> Maybe (Value a)) builtinBinary name = do Refl <- testEquality (typeRep @c) (typeRep @b) case (comparison, arithmetic, integral, textEquality, textContains) of@@ -399,8 +399,8 @@ Refl <- testEquality (typeRep @a) (typeRep @Int) Refl <- testEquality (typeRep @c) (typeRep @Int) op <- case name of- "div" -> Just div- "mod" -> Just mod+ BDiv -> Just div+ BMod -> Just mod _ -> Nothing -- Only non-zero literal divisors: lazy evaluation might skip a zero. Just $ \l r -> case r of@@ -411,18 +411,18 @@ Refl <- testEquality (typeRep @c) (typeRep @T.Text) Refl <- testEquality (typeRep @a) (typeRep @Bool) case name of- "contains" -> Just packedContains+ BContains -> Just packedContains _ -> Nothing comparison :: (c ~ b) => Maybe (Value c -> Value c -> Maybe (Value a)) comparison = do Refl <- testEquality (typeRep @a) (typeRep @Bool)- op <- K.cmpOpByName name+ op <- K.cmpOpOf name run <- K.compareOperands @c Just (\l r -> fromOperand <$> (run op <$> operand l <*> operand r)) arithmetic :: (c ~ b) => Maybe (Value c -> Value c -> Maybe (Value a)) arithmetic = do Refl <- testEquality (typeRep @a) (typeRep @c)- op <- K.arithOpByName name+ op <- K.arithOpOf name run <- K.arithOperands @c case sUnbox @c of STrue -> Just (\l r -> fromOperand <$> (run op <$> operand l <*> operand r))@@ -432,8 +432,8 @@ Refl <- testEquality (typeRep @c) (typeRep @T.Text) Refl <- testEquality (typeRep @a) (typeRep @Bool) wantEq <- case name of- "eq" -> Just True- "neq" -> Just False+ BEq -> Just True+ BNeq -> Just False _ -> Nothing Just (packedEquals wantEq) @@ -458,14 +458,14 @@ -- | As 'builtinBinary', for unary operators. builtinUnary :: forall b a.- (Columnable b, Columnable a) => T.Text -> Maybe (Value b -> Maybe (Value a))-builtinUnary "toDouble"+ (Columnable b, Columnable a) => UnOpId -> Maybe (Value b -> Maybe (Value a))+builtinUnary UToDouble | Just Refl <- testEquality (typeRep @b) (typeRep @Int) , Just Refl <- testEquality (typeRep @a) (typeRep @Double) = Just $ \v -> case operand v of Just (K.Column x) -> Just (Flat (UnboxedColumn Nothing (K.intToDouble x))) _ -> Nothing-builtinUnary "textLength"+builtinUnary UTextLength | Just Refl <- testEquality (typeRep @b) (typeRep @T.Text) , Just Refl <- testEquality (typeRep @a) (typeRep @Int) = Just $ \case@@ -477,15 +477,15 @@ forall op b a. (UnaryOp op, Columnable b, Columnable a) => op b a -> Value b -> Either DataFrameException (Value a)-applyUnary op v = case builtinUnary @b @a (unaryName op) >>= ($ v) of+applyUnary op v = case builtinOf (unaryId op) >>= builtinUnary @b @a >>= ($ v) of Just out -> Right out- Nothing -> liftValue (fastUnaryFn @b @a (unaryName op) (unaryFn op)) v+ Nothing -> liftValue (fastUnaryFn @b @a (unaryId op) (unaryFn op)) v applyBinary :: forall op c b a. (BinaryOp op, Columnable c, Columnable b, Columnable a) => op c b a -> Value c -> Value b -> Either DataFrameException (Value a)-applyBinary op l r = case builtinBinary @c @b @a (binaryName op) >>= (\run -> run l r) of+applyBinary op l r = case builtinOf (binaryId op) >>= builtinBinary @c @b @a >>= (\run -> run l r) of Just out -> Right out Nothing -> liftValue2 (binaryFn op) l r @@ -519,8 +519,8 @@ data Key = KeyColumn T.Text SomeTypeRep | KeyLiteral Literal- | KeyUnary T.Text SomeTypeRep Key- | KeyBinary T.Text SomeTypeRep Key Key+ | KeyUnary UnOpId SomeTypeRep Key+ | KeyBinary BinOpId SomeTypeRep Key Key deriving (Eq, Ord) data Literal@@ -544,13 +544,15 @@ Col name -> Just (KeyColumn name (SomeTypeRep (typeRep @x)), budget - 1) Lit v -> (\l -> (KeyLiteral l, budget - 1)) <$> literalKey v Unary op (inner :: Expr b) -> do- _ <- builtinUnary @b @x (unaryName op)+ tag <- builtinOf (unaryId op)+ _ <- builtinUnary @b @x tag (k, left) <- go (budget - 1) inner- Just (KeyUnary (unaryName op) (SomeTypeRep (typeRep @b)) k, left)+ Just (KeyUnary tag (SomeTypeRep (typeRep @b)) k, left) Binary op (l :: Expr c) (r :: Expr b) -> do- _ <- builtinBinary @c @b @x (binaryName op)+ tag <- builtinOf (binaryId op)+ _ <- builtinBinary @c @b @x tag -- A division is shareable only with a non-zero literal divisor.- if binaryName op `elem` ["div", "mod"]+ if tag `elem` [BDiv, BMod] then case r of Lit (k :: b) | Just Refl <- testEquality (typeRep @b) (typeRep @Int)@@ -560,7 +562,7 @@ else Just () (kl, afterLeft) <- go (budget - 1) l (kr, left) <- go afterLeft r- Just (KeyBinary (binaryName op) (SomeTypeRep (typeRep @c)) kl kr, left)+ Just (KeyBinary tag (SomeTypeRep (typeRep @c)) kl kr, left) _ -> Nothing literalKey :: forall a. (Columnable a) => a -> Maybe Literal@@ -1181,7 +1183,7 @@ Just col -> Flat <$> foldl1DirectGroups @b f col (valueIndices gdf) (offsets gdf) -- Fast path: countDistinct on a bare Col goes to the partitioned hash kernel.-eval ctx (Agg (MergeAgg "countDistinct" _ _ _ _) (Col name))+eval ctx (Agg (MergeAgg (Builtin ACountDistinct) _ _ _ _) (Col name)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) , Just col <- getColumn name (ctxFrame ctx) , Just v <- distinctValue col =@@ -1238,7 +1240,7 @@ Group gs -> Flat . fromVector <$> V.mapM (foldlColumn @b @a f seed) gs-eval (FlatCtx df) (Agg (MergeAgg "mean" _ _ _ (_ :: acc -> a)) (Col name))+eval (FlatCtx df) (Agg (MergeAgg (Builtin AMean) _ _ _ (_ :: acc -> a)) (Col name)) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) , Just v <- fusedMean df name = Right (Scalar v)@@ -1265,7 +1267,7 @@ Flat . fromVector <$> V.mapM (fmap finalize . foldlColumn @b step seed) gs -- Fast path: whole-frame sum of a clean Int/Double column, one parallel loop.-eval (FlatCtx df) (Agg (FoldAgg "sum" Nothing (_ :: a -> b -> a)) inner)+eval (FlatCtx df) (Agg (FoldAgg (Builtin ASum) Nothing (_ :: a -> b -> a)) inner) | Just Refl <- testEquality (typeRep @a) (typeRep @b) , Just v <- fusedSum df inner = Right (Scalar v)@@ -1294,9 +1296,9 @@ fastUnaryFn :: forall b a. (Columnable b, Columnable a) =>- T.Text -> (b -> a) -> (b -> a)+ OpRef UnOpId -> (b -> a) -> (b -> a) fastUnaryFn name f- | name == "toDouble"+ | Builtin UToDouble <- name , Just Refl <- testEquality (typeRep @a) (typeRep @Double) = integralToDouble @b f | otherwise = f@@ -1394,15 +1396,15 @@ Binary op (Col name :: Expr c) (Lit (k :: b)) | Just Refl <- testEquality (typeRep @c) (typeRep @a) , Just Refl <- testEquality (typeRep @b) (typeRep @a) ->- withClean name (Just (binaryName op, k, False))+ builtinOf (binaryId op) >>= \tag -> withClean name (Just (tag, k, False)) Binary op (Lit (k :: c)) (Col name :: Expr b) | Just Refl <- testEquality (typeRep @c) (typeRep @a) , Just Refl <- testEquality (typeRep @b) (typeRep @a) ->- withClean name (Just (binaryName op, k, True))+ builtinOf (binaryId op) >>= \tag -> withClean name (Just (tag, k, True)) _ -> Nothing where- -- The operator is picked by name at a concrete element type.- withClean :: T.Text -> Maybe (T.Text, a, Bool) -> Maybe a+ -- The operator is picked by tag at a concrete element type.+ withClean :: T.Text -> Maybe (BinOpId, a, Bool) -> Maybe a withClean name combo = do UnboxedColumn Nothing (v :: VU.Vector t) <- getColumn name df Refl <- testEquality (typeRep @t) (typeRep @a)@@ -1412,20 +1414,20 @@ Just Refl -> sumDouble combo v Nothing -> Nothing -sumInt :: Maybe (T.Text, Int, Bool) -> VU.Vector Int -> Maybe Int+sumInt :: Maybe (BinOpId, Int, Bool) -> VU.Vector Int -> Maybe Int sumInt combo v = case combo of Nothing -> Just (K.parSumBy id v)- Just ("add", k, _) -> Just (K.parSumBy (+ k) v)- Just ("mult", k, _) -> Just (K.parSumBy (* k) v)- Just ("sub", k, False) -> Just (K.parSumBy (subtract k) v)- Just ("sub", k, True) -> Just (K.parSumBy (k -) v)+ Just (BAdd, k, _) -> Just (K.parSumBy (+ k) v)+ Just (BMult, k, _) -> Just (K.parSumBy (* k) v)+ Just (BSub, k, False) -> Just (K.parSumBy (subtract k) v)+ Just (BSub, k, True) -> Just (K.parSumBy (k -) v) _ -> Nothing -sumDouble :: Maybe (T.Text, Double, Bool) -> VU.Vector Double -> Maybe Double+sumDouble :: Maybe (BinOpId, Double, Bool) -> VU.Vector Double -> Maybe Double sumDouble combo v = case combo of Nothing -> Just (K.parSumBy id v)- Just ("add", k, _) -> Just (K.parSumBy (+ k) v)- Just ("mult", k, _) -> Just (K.parSumBy (* k) v)- Just ("sub", k, False) -> Just (K.parSumBy (subtract k) v)- Just ("sub", k, True) -> Just (K.parSumBy (k -) v)+ Just (BAdd, k, _) -> Just (K.parSumBy (+ k) v)+ Just (BMult, k, _) -> Just (K.parSumBy (* k) v)+ Just (BSub, k, False) -> Just (K.parSumBy (subtract k) v)+ Just (BSub, k, True) -> Just (K.parSumBy (k -) v) _ -> Nothing
src-internal/DataFrame/Internal/Interpreter/Kernels.hs view
@@ -1,6 +1,5 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE GADTs #-}-{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}@@ -10,9 +9,9 @@ Operand (..), Branch (..), CmpOp (..),- cmpOpByName,+ cmpOpOf, ArithOp (..),- arithOpByName,+ arithOpOf, compareOperands, arithOperands, zipOperands,@@ -44,6 +43,7 @@ selAt, sliceEqBytes, )+import DataFrame.Internal.Expression.OpId (BinOpId (..)) import System.IO.Unsafe (unsafePerformIO) -- | One side of an operation: the same value in every row, or one per row.@@ -52,24 +52,24 @@ data CmpOp = CmpLt | CmpLeq | CmpGt | CmpGeq | CmpEq | CmpNeq deriving (Eq, Show) -cmpOpByName :: T.Text -> Maybe CmpOp-cmpOpByName name = case name of- "lt" -> Just CmpLt- "leq" -> Just CmpLeq- "gt" -> Just CmpGt- "geq" -> Just CmpGeq- "eq" -> Just CmpEq- "neq" -> Just CmpNeq+cmpOpOf :: BinOpId -> Maybe CmpOp+cmpOpOf op = case op of+ BLt -> Just CmpLt+ BLeq -> Just CmpLeq+ BGt -> Just CmpGt+ BGeq -> Just CmpGeq+ BEq -> Just CmpEq+ BNeq -> Just CmpNeq _ -> Nothing data ArithOp = ArithAdd | ArithSub | ArithMult deriving (Eq, Show) -arithOpByName :: T.Text -> Maybe ArithOp-arithOpByName name = case name of- "add" -> Just ArithAdd- "sub" -> Just ArithSub- "mult" -> Just ArithMult+arithOpOf :: BinOpId -> Maybe ArithOp+arithOpOf op = case op of+ BAdd -> Just ArithAdd+ BSub -> Just ArithSub+ BMult -> Just ArithMult _ -> Nothing -- | Runs @k@ with the element type known, only for 'Double' or 'Int'.
src/DataFrame/Typed/Schema.hs view
@@ -34,6 +34,7 @@ AssertAllPresent, AssertKeyTypesMatch, AssertDisjoint,+ AssertAllAbsent, AssertAllColumnsHaveType, AssertRealColumn, AllColumnsReal,@@ -200,6 +201,15 @@ type family ColumnNames (cols :: [(Symbol, Type)]) :: [Symbol] where ColumnNames '[] = '[] ColumnNames ('(n, _) ': rest) = n ': ColumnNames rest++-- | Assert that none of the new columns already exists in the schema.+type family+ AssertAllAbsent (new :: [(Symbol, Type)]) (cols :: [(Symbol, Type)]) ::+ Constraint+ where+ AssertAllAbsent '[] cols = ()+ AssertAllAbsent ('(n, _) ': rest) cols =+ (AssertAbsent n cols, AssertAllAbsent rest cols) -- | Assert that a column name is absent from the schema (for derive/insert). type family AssertAbsent (name :: Symbol) (cols :: [(Symbol, Type)]) :: Constraint where