packages feed

dataframe 0.4.0.3 → 0.4.0.4

raw patch · 9 files changed

+268/−98 lines, 9 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

- DataFrame.DecisionTree: [maxDepth] :: TreeConfig -> Int
- DataFrame.DecisionTree: generateConditions :: DataFrame -> [Expr Bool]
+ DataFrame: genericPercentile :: (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a
+ DataFrame: medianMaybe :: (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double
+ DataFrame: percentile :: (Columnable a, Real a, Unbox a) => Int -> Expr a -> DataFrame -> Double
+ DataFrame.DecisionTree: SynthConfig :: Int -> [Int] -> Bool -> Bool -> Bool -> SynthConfig
+ DataFrame.DecisionTree: [enableArithOps] :: SynthConfig -> Bool
+ DataFrame.DecisionTree: [enableCrossCols] :: SynthConfig -> Bool
+ DataFrame.DecisionTree: [enableStringOps] :: SynthConfig -> Bool
+ DataFrame.DecisionTree: [maxExprDepth] :: SynthConfig -> Int
+ DataFrame.DecisionTree: [maxTreeDepth] :: TreeConfig -> Int
+ DataFrame.DecisionTree: [percentiles] :: SynthConfig -> [Int]
+ DataFrame.DecisionTree: [synthConfig] :: TreeConfig -> SynthConfig
+ DataFrame.DecisionTree: boolExprs :: DataFrame -> [Expr Bool] -> [Expr Bool] -> Int -> Int -> [Expr Bool]
+ DataFrame.DecisionTree: data SynthConfig
+ DataFrame.DecisionTree: defaultSynthConfig :: SynthConfig
+ DataFrame.DecisionTree: defaultTreeConfig :: TreeConfig
+ DataFrame.DecisionTree: generateConditionsOld :: SynthConfig -> DataFrame -> [Expr Bool]
+ DataFrame.DecisionTree: generateNumericConds :: SynthConfig -> DataFrame -> [Expr Bool]
+ DataFrame.DecisionTree: numericCols :: DataFrame -> [Expr Double]
+ DataFrame.DecisionTree: numericConditions :: CondGen
+ DataFrame.DecisionTree: numericExprs :: SynthConfig -> DataFrame -> [Expr Double] -> Int -> Int -> [Expr Double]
+ DataFrame.DecisionTree: numericExprsWithTerms :: SynthConfig -> DataFrame -> [Expr Double]
+ DataFrame.DecisionTree: type CondGen = SynthConfig -> DataFrame -> [Expr Bool]
+ DataFrame.Functions: fromJust :: Columnable a => Expr (Maybe a) -> Expr a
+ DataFrame.Functions: fromMaybe :: Columnable a => a -> Expr (Maybe a) -> Expr a
+ DataFrame.Functions: isJust :: Columnable a => Expr (Maybe a) -> Expr Bool
+ DataFrame.Functions: isNothing :: Columnable a => Expr (Maybe a) -> Expr Bool
+ DataFrame.Functions: recodeWithCondition :: (Columnable a, Columnable b) => Expr b -> [(Expr a -> Expr Bool, b)] -> Expr a -> Expr b
+ DataFrame.Functions: toMaybe :: Columnable a => Expr a -> Expr (Maybe a)
+ DataFrame.Internal.Statistics: meanInt' :: Vector Int -> Double
+ DataFrame.Operations.Statistics: genericPercentile :: (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a
+ DataFrame.Operations.Statistics: medianMaybe :: (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double
+ DataFrame.Operations.Statistics: percentile :: (Columnable a, Real a, Unbox a) => Int -> Expr a -> DataFrame -> Double
- DataFrame.DecisionTree: TreeConfig :: Int -> Int -> TreeConfig
+ DataFrame.DecisionTree: TreeConfig :: Int -> Int -> SynthConfig -> TreeConfig

Files

CHANGELOG.md view
@@ -1,5 +1,14 @@ # Revision history for dataframe +## 0.4.0.4+* More robust synthesis based decision tree+* Improved performance on sum and mean.+* recodeWithCondition - to change a value given a condition+* medianMaybe, genericPercentile, percentile - self explanatory+* all the maybe functions as dataframe functions+* Fix concatColumnsEither when types are the same.+* Decision tree implementation is more robust now.+ ## 0.4.0.3 * Improved performance for folds and reductions. * Improve standalone mean and correlation functions.
dataframe.cabal view
@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.4.0.3+version:            0.4.0.4  synopsis: A fast, safe, and intuitive DataFrame library. 
src/DataFrame.hs view
@@ -307,11 +307,14 @@ import DataFrame.Operations.Statistics as Statistics (     correlation,     frequencies,+    genericPercentile,     imputeWith,     interQuartileRange,     mean,     meanMaybe,     median,+    medianMaybe,+    percentile,     skewness,     standardDeviation,     sum,
src/DataFrame/DecisionTree.hs view
@@ -1,7 +1,10 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE BangPatterns #-}+{-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-}+{-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}@@ -11,14 +14,18 @@ import qualified DataFrame.Functions as F import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn)-import DataFrame.Internal.Expression (Expr (..), interpret)+import DataFrame.Internal.Expression (Expr (..), eSize, interpret) import DataFrame.Internal.Statistics (percentileOrd')+import DataFrame.Internal.Types import DataFrame.Operations.Core (columnNames, nRows)+import DataFrame.Operations.Statistics (percentile) import DataFrame.Operations.Subset (exclude, filterWhere)  import Control.Exception (throw)+import Control.Monad (guard)+import Data.Containers.ListUtils (nubOrd) import Data.Function (on)-import Data.List (foldl', maximumBy)+import Data.List (foldl', maximumBy, sortBy) import qualified Data.Map.Strict as M import qualified Data.Text as T import Data.Type.Equality@@ -26,13 +33,40 @@ import qualified Data.Vector.Unboxed as VU import Type.Reflection (typeRep) -import DataFrame.Functions ((.<=), (.==))+import DataFrame.Functions ((.<), (.<=), (.==), (.>), (.>=))  data TreeConfig = TreeConfig-    { maxDepth :: Int+    { maxTreeDepth :: Int     , minSamplesSplit :: Int+    , synthConfig :: SynthConfig     } +data SynthConfig = SynthConfig+    { maxExprDepth :: Int+    , percentiles :: [Int]+    , enableStringOps :: Bool+    , enableCrossCols :: Bool+    , enableArithOps :: Bool+    }++defaultSynthConfig :: SynthConfig+defaultSynthConfig =+    SynthConfig+        { maxExprDepth = 2+        , percentiles = [0, 10 .. 100]+        , enableStringOps = True+        , enableCrossCols = True+        , enableArithOps = True+        }++defaultTreeConfig :: TreeConfig+defaultTreeConfig =+    TreeConfig+        { maxTreeDepth = 10+        , minSamplesSplit = 5+        , synthConfig = defaultSynthConfig+        }+ fitDecisionTree ::     forall a.     (Columnable a) =>@@ -43,9 +77,11 @@ fitDecisionTree cfg (Col target) df =     buildTree @a         cfg-        (maxDepth cfg)+        (maxTreeDepth cfg)         target-        (generateConditions (exclude [target] df))+        ( numericConditions (synthConfig cfg) (exclude [target] df)+            ++ generateConditionsOld (synthConfig cfg) (exclude [target] df)+        )         df fitDecisionTree _ expr _ = error $ "Cannot create tree for compound expression: " ++ show expr @@ -87,6 +123,7 @@             else case (t, f) of                 -- Nested simplification: `if C1 then (if C1 then X else Y) else Z`                 -- becomes:     if C1 then X else Z`+                -- Generalize this with hegg later.                 (If condInner tInner fInner, _) | cond == condInner -> If cond tInner f                 (_, If condInner tInner fInner) | cond == condInner -> If cond t fInner                 _ -> If cond t f@@ -94,8 +131,75 @@ pruneTree (BinaryOp name op l r) = BinaryOp name op (pruneTree l) (pruneTree r) pruneTree e = e -generateConditions :: DataFrame -> [Expr Bool]-generateConditions df =+type CondGen = SynthConfig -> DataFrame -> [Expr Bool]++numericConditions :: CondGen+numericConditions = generateNumericConds++generateNumericConds ::+    SynthConfig -> DataFrame -> [Expr Bool]+generateNumericConds cfg df = do+    expr <- numericExprsWithTerms cfg df+    let thresholds = map (\p -> percentile p expr df) (percentiles cfg)+    threshold <- thresholds+    [ expr .<= F.lit threshold+        , expr .>= F.lit threshold+        , expr .< F.lit threshold+        , expr .> F.lit threshold+        ]++numericExprsWithTerms ::+    SynthConfig -> DataFrame -> [Expr Double]+numericExprsWithTerms cfg df =+    concatMap (numericExprs cfg df [] 0) [0 .. maxExprDepth cfg]++numericCols :: DataFrame -> [Expr Double]+numericCols df = concatMap extract (columnNames df)+  where+    extract col = case unsafeGetColumn col df of+        UnboxedColumn (_ :: VU.Vector b) ->+            case testEquality (typeRep @b) (typeRep @Double) of+                Just Refl -> [Col col]+                Nothing -> case sIntegral @b of+                    STrue -> [F.toDouble (Col @b col)]+                    SFalse -> []+        _ -> []++numericExprs ::+    SynthConfig -> DataFrame -> [Expr Double] -> Int -> Int -> [Expr Double]+numericExprs cfg df prevExprs depth maxDepth+    | depth == 0 = baseExprs ++ numericExprs cfg df baseExprs (depth + 1) maxDepth+    | depth >= maxDepth = []+    | otherwise =+        combinedExprs ++ numericExprs cfg df combinedExprs (depth + 1) maxDepth+  where+    baseExprs = numericCols df++    combinedExprs+        | not (enableArithOps cfg) = []+        | otherwise = do+            e1 <- prevExprs+            e2 <- baseExprs+            guard (e1 /= e2)+            [e1 + e2, e1 - e2, e1 * e2, F.ifThenElse (e2 .>= 0) (e1 / e2) 0]++boolExprs ::+    DataFrame -> [Expr Bool] -> [Expr Bool] -> Int -> Int -> [Expr Bool]+boolExprs df baseExprs prevExprs depth maxDepth+    | depth == 0 =+        baseExprs ++ boolExprs df baseExprs prevExprs (depth + 1) maxDepth+    | depth >= maxDepth = []+    | otherwise =+        combinedExprs ++ boolExprs df baseExprs combinedExprs (depth + 1) maxDepth+  where+    combinedExprs = do+        e1 <- prevExprs+        e2 <- baseExprs+        guard (e1 /= e2)+        [F.and e1 e2, F.or e1 e2]++generateConditionsOld :: SynthConfig -> DataFrame -> [Expr Bool]+generateConditionsOld cfg df =     let         genConds :: T.Text -> [Expr Bool]         genConds colName = case unsafeGetColumn colName df of@@ -103,32 +207,27 @@                 let                     percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99]                  in-                    map (Col @a colName .<=) percentiles-                        ++ map (Col @a colName .==) percentiles+                    map (Col @a colName .==) percentiles             (OptionalColumn (col :: V.Vector a)) ->                 let                     percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99]                  in-                    map (Col @a colName .<=) percentiles-                        ++ map (Col @a colName .==) percentiles-            (UnboxedColumn (col :: VU.Vector a)) ->-                let-                    percentiles = map (Lit . (`percentileOrd'` VU.convert col)) [1, 25, 75, 99]-                 in-                    map (Col @a colName .<=) percentiles-                        ++ map (Col @a colName .==) percentiles+                    map (Col @a colName .==) percentiles+            (UnboxedColumn (col :: VU.Vector a)) -> []         columnConds = concatMap colConds [(l, r) | l <- columnNames df, r <- columnNames df]           where             colConds (!l, !r) = case (unsafeGetColumn l df, unsafeGetColumn r df) of                 (BoxedColumn (col1 :: V.Vector a), BoxedColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of                     Nothing -> []                     Just Refl -> [Col @a l .== Col @a r]-                (UnboxedColumn (col1 :: VU.Vector a), UnboxedColumn (col2 :: VU.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of-                    Nothing -> []-                    Just Refl -> [Col @a l .<= Col @a r, Col @a l .== Col @a r]-                (OptionalColumn (col1 :: V.Vector a), OptionalColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of-                    Nothing -> []-                    Just Refl -> [Col @a l .<= Col @a r, Col @a l .== Col @a r]+                (UnboxedColumn (col1 :: VU.Vector a), UnboxedColumn (col2 :: VU.Vector b)) -> []+                ( OptionalColumn (col1 :: V.Vector (Maybe a))+                    , OptionalColumn (col2 :: V.Vector (Maybe b))+                    ) -> case testEquality (typeRep @a) (typeRep @b) of+                        Nothing -> []+                        Just Refl -> case testEquality (typeRep @a) (typeRep @T.Text) of+                            Nothing -> [Col @(Maybe a) l .<= Col r, Col @(Maybe a) l .== Col r]+                            Just Refl -> [Col @(Maybe a) l .== Col r]                 _ -> []      in         concatMap genConds (columnNames df) ++ columnConds@@ -142,6 +241,8 @@     T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool) findBestSplit target conds df =     let+        minLeafSize = 1+        lambda = 0.05         initialImpurity = calculateGini @a target df         evalGain cond =             let (t, f) = partitionDataFrame cond df@@ -151,13 +252,28 @@                 newImpurity =                     (weightT * calculateGini @a target t)                         + (weightF * calculateGini @a target f)-             in initialImpurity - newImpurity+             in ( (initialImpurity - newImpurity) - lambda * fromIntegral (eSize cond)+                , negate (eSize cond)+                ) -        validConds = filter (\c -> nRows (filterWhere c df) > 0) conds+        validConds =+            filter+                ( \c ->+                    let+                        (t, f) = partitionDataFrame c df+                     in+                        nRows t >= minLeafSize && nRows f >= minLeafSize+                )+                (nubOrd conds)+        sortedConditions = take 10 (sortBy (flip compare `on` evalGain) validConds)      in         if null validConds             then Nothing-            else Just $ maximumBy (compare `on` evalGain) validConds+            else+                Just $+                    maximumBy+                        (compare `on` evalGain)+                        (boolExprs df sortedConditions sortedConditions 0 3)  calculateGini ::     forall a.@@ -166,7 +282,8 @@ calculateGini target df =     let n = fromIntegral $ nRows df         counts = getCounts @a target df-        probs = map (\c -> fromIntegral c / n) (M.elems counts)+        numClasses = fromIntegral $ M.size counts+        probs = map (\c -> (fromIntegral c + 1) / (n + numClasses)) (M.elems counts)      in if n == 0 then 0 else 1 - sum (map (^ 2) probs)  majorityValue ::
src/DataFrame/Functions.hs view
@@ -33,7 +33,7 @@ import Data.Functor import qualified Data.List as L import qualified Data.Map as M-import Data.Maybe (catMaybes, fromMaybe, isJust, listToMaybe)+import qualified Data.Maybe as Maybe import qualified Data.Text as T import Data.Time import qualified Data.Vector as V@@ -107,19 +107,13 @@  -- TODO: Generalize this pattern for other equality functions. (.>) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(.>) (Lit l) (Lit r) = Lit (l > r)-(.>) (Lit l) expr = UnaryOp ("gt " <> T.pack (show l)) (l >) expr-(.>) expr (Lit r) = UnaryOp ("leq " <> T.pack (show r)) (r <=) expr-(.>) l r = BinaryOp "gt" (>) l r+(.>) = BinaryOp "gt" (>)  gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool gt = (.>)  (.<=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-(.<=) (Lit l) (Lit r) = Lit (l <= r)-(.<=) (Lit l) expr = UnaryOp ("leq " <> T.pack (show l)) (l <=) expr-(.<=) expr (Lit r) = UnaryOp ("gt " <> T.pack (show r)) (r >) expr-(.<=) l r = BinaryOp "leq" (<=) l r+(.<=) = BinaryOp "leq" (<=)  leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool leq = (.<=)@@ -170,12 +164,18 @@  sum :: forall a. (Columnable a, Num a) => Expr a -> Expr a sum expr = AggReduce expr "sum" (+)+{-# SPECIALIZE DataFrame.Functions.sum :: Expr Double -> Expr Double #-}+{-# SPECIALIZE DataFrame.Functions.sum :: Expr Int -> Expr Int #-}+{-# INLINEABLE DataFrame.Functions.sum #-}  sumMaybe :: forall a. (Columnable a, Num a) => Expr (Maybe a) -> Expr a-sumMaybe expr = AggVector expr "sumMaybe" (P.sum . catMaybes . V.toList)+sumMaybe expr = AggVector expr "sumMaybe" (P.sum . Maybe.catMaybes . V.toList)  mean :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double mean expr = AggNumericVector expr "mean" mean'+{-# SPECIALIZE DataFrame.Functions.mean :: Expr Double -> Expr Double #-}+{-# SPECIALIZE DataFrame.Functions.mean :: Expr Int -> Expr Double #-}+{-# INLINEABLE DataFrame.Functions.mean #-}  meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double meanMaybe expr = AggVector expr "meanMaybe" (mean' . optionalToDoubleVector)@@ -193,7 +193,7 @@ optionalToDoubleVector =     VU.fromList         . V.foldl'-            (\acc e -> if isJust e then realToFrac (fromMaybe 0 e) : acc else acc)+            (\acc e -> if Maybe.isJust e then realToFrac (Maybe.fromMaybe 0 e) : acc else acc)             []  percentile :: Int -> Expr Double -> Expr Double@@ -232,6 +232,21 @@     (Columnable a, Columnable b) => Expr b -> a -> (a -> b -> a) -> Expr a reduce expr = AggFold expr "foldUdf" +toMaybe :: (Columnable a) => Expr a -> Expr (Maybe a)+toMaybe = UnaryOp "toMaybe" Just++fromMaybe :: (Columnable a) => a -> Expr (Maybe a) -> Expr a+fromMaybe d = UnaryOp ("fromMaybe " <> T.pack (show d)) (Maybe.fromMaybe d)++isJust :: (Columnable a) => Expr (Maybe a) -> Expr Bool+isJust = UnaryOp "isJust" Maybe.isJust++isNothing :: (Columnable a) => Expr (Maybe a) -> Expr Bool+isNothing = UnaryOp "isNothing" Maybe.isNothing++fromJust :: (Columnable a) => Expr (Maybe a) -> Expr a+fromJust = UnaryOp "fromJust" Maybe.fromJust+ whenPresent ::     forall a b.     (Columnable a, Columnable b) => (a -> b) -> Expr (Maybe a) -> Expr (Maybe b)@@ -248,17 +263,26 @@     (Columnable a, Columnable b) => [(a, b)] -> Expr a -> Expr (Maybe b) recode mapping = UnaryOp (T.pack ("recode " ++ show mapping)) (`lookup` mapping) +recodeWithCondition ::+    forall a b.+    (Columnable a, Columnable b) =>+    Expr b -> [(Expr a -> Expr Bool, b)] -> Expr a -> Expr b+recodeWithCondition fallback [] value = fallback+recodeWithCondition fallback ((cond, value) : rest) expr = ifThenElse (cond expr) (lit value) (recodeWithCondition fallback rest expr)+ recodeWithDefault ::     forall a b.     (Columnable a, Columnable b) => b -> [(a, b)] -> Expr a -> Expr b recodeWithDefault d mapping =-    UnaryOp (T.pack ("recode " ++ show mapping)) (fromMaybe d . (`lookup` mapping))+    UnaryOp+        (T.pack ("recode " ++ show mapping))+        (Maybe.fromMaybe d . (`lookup` mapping))  firstOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)-firstOrNothing = lift listToMaybe+firstOrNothing = lift Maybe.listToMaybe  lastOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)-lastOrNothing = lift (listToMaybe . reverse)+lastOrNothing = lift (Maybe.listToMaybe . reverse)  splitOn :: T.Text -> Expr T.Text -> Expr [T.Text] splitOn delim = lift (T.splitOn delim)
src/DataFrame/Internal/Column.hs view
@@ -1023,18 +1023,24 @@     BoxedColumn $ fmap Left left <> fmap Right (VG.convert right) concatColumnsEither (UnboxedColumn left) (BoxedColumn right) =     BoxedColumn $ fmap Left (VG.convert left) <> fmap Right right-concatColumnsEither (OptionalColumn left) (BoxedColumn right) =-    OptionalColumn $-        fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right)-concatColumnsEither (BoxedColumn left) (OptionalColumn right) =-    OptionalColumn $-        fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right-concatColumnsEither (OptionalColumn left) (UnboxedColumn right) =-    OptionalColumn $-        fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right)-concatColumnsEither (UnboxedColumn left) (OptionalColumn right) =-    OptionalColumn $-        fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right+concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (BoxedColumn (right :: VB.Vector b)) =+    case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> OptionalColumn $ left <> fmap Just right+        Nothing -> OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) right+concatColumnsEither (BoxedColumn (left :: VB.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) =+    case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> OptionalColumn $ fmap Just left <> right+        Nothing -> OptionalColumn $ fmap (Just . Left) left <> fmap (fmap Right) right+concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (UnboxedColumn (right :: VU.Vector b)) =+    case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> OptionalColumn $ left <> fmap Just (VG.convert right)+        Nothing ->+            OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right)+concatColumnsEither (UnboxedColumn (left :: VU.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) =+    case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> OptionalColumn $ fmap Just (VG.convert left) <> right+        Nothing ->+            OptionalColumn $ fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right  {- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified. 
src/DataFrame/Internal/Statistics.hs view
@@ -18,13 +18,28 @@ mean' samp     | VU.null samp = throw $ EmptyDataSetException "mean"     | otherwise = rtf (VU.sum samp) / fromIntegral (VU.length samp)-{-# INLINE mean' #-}+{-# INLINE [0] mean' #-}  meanDouble' :: VU.Vector Double -> Double meanDouble' samp     | VU.null samp = throw $ EmptyDataSetException "mean"     | otherwise = VU.sum samp / fromIntegral (VU.length samp) {-# INLINE meanDouble' #-}++meanInt' :: VU.Vector Int -> Double+meanInt' samp+    | VU.null samp = throw $ EmptyDataSetException "mean"+    | otherwise = fromIntegral (VU.sum samp) / fromIntegral (VU.length samp)+{-# INLINE meanInt' #-}++{-# RULES+"mean'/Double" [1] forall (xs :: VU.Vector Double).+    mean' xs =+        meanDouble' xs+"mean'/Int" [1] forall (xs :: VU.Vector Int).+    mean' xs =+        meanInt' xs+    #-}  median' :: (Real a, VU.Unbox a) => VU.Vector a -> Double median' samp
src/DataFrame/Operations/Statistics.hs view
@@ -119,6 +119,44 @@         Left e -> throw e         Right xs -> median' xs +-- | Calculates the median of a given column (containing optional values) as a standalone value.+medianMaybe ::+    forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double+medianMaybe (Col name) df =+    (median' . optionalToDoubleVector)+        (either throw id (columnAsVector (Col @(Maybe a) name) df))+medianMaybe expr df = case interpret @(Maybe a) df expr of+    Left e -> throw e+    Right (TColumn col) -> case toVector @(Maybe a) col of+        Left e -> throw e+        Right xs -> (median' . optionalToDoubleVector) xs++-- | Calculates the nth percentile of a given column as a standalone value.+percentile ::+    forall a.+    (Columnable a, Real a, VU.Unbox a) => Int -> Expr a -> DataFrame -> Double+percentile n (Col name) df = case columnAsUnboxedVector (Col @a name) df of+    Right xs -> percentile' n xs+    Left e -> throw e+percentile n expr df = case interpret df expr of+    Left e -> throw e+    Right (TColumn col) -> case toUnboxedVector @a col of+        Left e -> throw e+        Right xs -> percentile' n xs++-- | Calculates the nth percentile of a given column as a standalone value.+genericPercentile ::+    forall a.+    (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a+genericPercentile n (Col name) df = case columnAsVector (Col @a name) df of+    Right xs -> percentileOrd' n xs+    Left e -> throw e+genericPercentile n expr df = case interpret df expr of+    Left e -> throw e+    Right (TColumn col) -> case toVector @a col of+        Left e -> throw e+        Right xs -> percentileOrd' n xs+ -- | Calculates the standard deviation of a given column as a standalone value. standardDeviation ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double
tests/Functions.hs view
@@ -6,11 +6,9 @@ import qualified DataFrame as D import DataFrame.Functions (     sanitize,-    (.>),  ) import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI-import DataFrame.Internal.Expression import Test.HUnit  -- Test cases for the sanitize function@@ -81,48 +79,8 @@             (D.derive "sum" (F.sum (F.col @Int "A")) df)         ) -testGtTwoLitsRewriteAsLit :: Test-testGtTwoLitsRewriteAsLit =-    TestCase-        ( assertEqual-            "Two lits rewrite as one"-            (Lit True)-            (Lit (5 :: Int) .> Lit 4)-        )--testGtLeftLitRewriteAsUnary :: Test-testGtLeftLitRewriteAsUnary =-    TestCase-        ( assertEqual-            "Two lits rewrite as single literal"-            (UnaryOp "gt 5" ((5 :: Int) >) (Col "x"))-            (Lit (5 :: Int) .> Col "x")-        )--testGtRightLitRewriteAsUnary :: Test-testGtRightLitRewriteAsUnary =-    TestCase-        ( assertEqual-            "Two lits rewrite as unary op"-            (UnaryOp "gt 5" ((5 :: Int) >) (Col "x"))-            (Lit (5 :: Int) .> Col "x")-        )--testGtNoLitWriteAsBinary :: Test-testGtNoLitWriteAsBinary =-    TestCase-        ( assertEqual-            "Two lits rewrite as unary op"-            (BinaryOp "gt" (>) (Col @Int "x") (Col "y"))-            (Col @Int "x" .> Col "y")-        )- tests :: [Test] tests =     [ TestLabel "sanitizeIdentifiers" sanitizeIdentifiers     , TestLabel "testSum" testSum-    , TestLabel "testGtTwoLitRewrite" testGtTwoLitsRewriteAsLit-    , TestLabel "testGtLeftLit" testGtLeftLitRewriteAsUnary-    , TestLabel "testGtRightLit" testGtRightLitRewriteAsUnary-    , TestLabel "testGtNoLit" testGtNoLitWriteAsBinary     ]