dataframe 0.4.0.3 → 0.4.0.4
raw patch · 9 files changed
+268/−98 lines, 9 filesPVP: major bump suggested
API removals or changes: PVP suggests a major version bump
API changes (from Hackage documentation)
- DataFrame.DecisionTree: [maxDepth] :: TreeConfig -> Int
- DataFrame.DecisionTree: generateConditions :: DataFrame -> [Expr Bool]
+ DataFrame: genericPercentile :: (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a
+ DataFrame: medianMaybe :: (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double
+ DataFrame: percentile :: (Columnable a, Real a, Unbox a) => Int -> Expr a -> DataFrame -> Double
+ DataFrame.DecisionTree: SynthConfig :: Int -> [Int] -> Bool -> Bool -> Bool -> SynthConfig
+ DataFrame.DecisionTree: [enableArithOps] :: SynthConfig -> Bool
+ DataFrame.DecisionTree: [enableCrossCols] :: SynthConfig -> Bool
+ DataFrame.DecisionTree: [enableStringOps] :: SynthConfig -> Bool
+ DataFrame.DecisionTree: [maxExprDepth] :: SynthConfig -> Int
+ DataFrame.DecisionTree: [maxTreeDepth] :: TreeConfig -> Int
+ DataFrame.DecisionTree: [percentiles] :: SynthConfig -> [Int]
+ DataFrame.DecisionTree: [synthConfig] :: TreeConfig -> SynthConfig
+ DataFrame.DecisionTree: boolExprs :: DataFrame -> [Expr Bool] -> [Expr Bool] -> Int -> Int -> [Expr Bool]
+ DataFrame.DecisionTree: data SynthConfig
+ DataFrame.DecisionTree: defaultSynthConfig :: SynthConfig
+ DataFrame.DecisionTree: defaultTreeConfig :: TreeConfig
+ DataFrame.DecisionTree: generateConditionsOld :: SynthConfig -> DataFrame -> [Expr Bool]
+ DataFrame.DecisionTree: generateNumericConds :: SynthConfig -> DataFrame -> [Expr Bool]
+ DataFrame.DecisionTree: numericCols :: DataFrame -> [Expr Double]
+ DataFrame.DecisionTree: numericConditions :: CondGen
+ DataFrame.DecisionTree: numericExprs :: SynthConfig -> DataFrame -> [Expr Double] -> Int -> Int -> [Expr Double]
+ DataFrame.DecisionTree: numericExprsWithTerms :: SynthConfig -> DataFrame -> [Expr Double]
+ DataFrame.DecisionTree: type CondGen = SynthConfig -> DataFrame -> [Expr Bool]
+ DataFrame.Functions: fromJust :: Columnable a => Expr (Maybe a) -> Expr a
+ DataFrame.Functions: fromMaybe :: Columnable a => a -> Expr (Maybe a) -> Expr a
+ DataFrame.Functions: isJust :: Columnable a => Expr (Maybe a) -> Expr Bool
+ DataFrame.Functions: isNothing :: Columnable a => Expr (Maybe a) -> Expr Bool
+ DataFrame.Functions: recodeWithCondition :: (Columnable a, Columnable b) => Expr b -> [(Expr a -> Expr Bool, b)] -> Expr a -> Expr b
+ DataFrame.Functions: toMaybe :: Columnable a => Expr a -> Expr (Maybe a)
+ DataFrame.Internal.Statistics: meanInt' :: Vector Int -> Double
+ DataFrame.Operations.Statistics: genericPercentile :: (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a
+ DataFrame.Operations.Statistics: medianMaybe :: (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double
+ DataFrame.Operations.Statistics: percentile :: (Columnable a, Real a, Unbox a) => Int -> Expr a -> DataFrame -> Double
- DataFrame.DecisionTree: TreeConfig :: Int -> Int -> TreeConfig
+ DataFrame.DecisionTree: TreeConfig :: Int -> Int -> SynthConfig -> TreeConfig
Files
- CHANGELOG.md +9/−0
- dataframe.cabal +1/−1
- src/DataFrame.hs +3/−0
- src/DataFrame/DecisionTree.hs +145/−28
- src/DataFrame/Functions.hs +38/−14
- src/DataFrame/Internal/Column.hs +18/−12
- src/DataFrame/Internal/Statistics.hs +16/−1
- src/DataFrame/Operations/Statistics.hs +38/−0
- tests/Functions.hs +0/−42
CHANGELOG.md view
@@ -1,5 +1,14 @@ # Revision history for dataframe +## 0.4.0.4+* More robust synthesis based decision tree+* Improved performance on sum and mean.+* recodeWithCondition - to change a value given a condition+* medianMaybe, genericPercentile, percentile - self explanatory+* all the maybe functions as dataframe functions+* Fix concatColumnsEither when types are the same.+* Decision tree implementation is more robust now.+ ## 0.4.0.3 * Improved performance for folds and reductions. * Improve standalone mean and correlation functions.
dataframe.cabal view
@@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe-version: 0.4.0.3+version: 0.4.0.4 synopsis: A fast, safe, and intuitive DataFrame library.
src/DataFrame.hs view
@@ -307,11 +307,14 @@ import DataFrame.Operations.Statistics as Statistics ( correlation, frequencies,+ genericPercentile, imputeWith, interQuartileRange, mean, meanMaybe, median,+ medianMaybe,+ percentile, skewness, standardDeviation, sum,
src/DataFrame/DecisionTree.hs view
@@ -1,7 +1,10 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE BangPatterns #-}+{-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-}+{-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}@@ -11,14 +14,18 @@ import qualified DataFrame.Functions as F import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn)-import DataFrame.Internal.Expression (Expr (..), interpret)+import DataFrame.Internal.Expression (Expr (..), eSize, interpret) import DataFrame.Internal.Statistics (percentileOrd')+import DataFrame.Internal.Types import DataFrame.Operations.Core (columnNames, nRows)+import DataFrame.Operations.Statistics (percentile) import DataFrame.Operations.Subset (exclude, filterWhere) import Control.Exception (throw)+import Control.Monad (guard)+import Data.Containers.ListUtils (nubOrd) import Data.Function (on)-import Data.List (foldl', maximumBy)+import Data.List (foldl', maximumBy, sortBy) import qualified Data.Map.Strict as M import qualified Data.Text as T import Data.Type.Equality@@ -26,13 +33,40 @@ import qualified Data.Vector.Unboxed as VU import Type.Reflection (typeRep) -import DataFrame.Functions ((.<=), (.==))+import DataFrame.Functions ((.<), (.<=), (.==), (.>), (.>=)) data TreeConfig = TreeConfig- { maxDepth :: Int+ { maxTreeDepth :: Int , minSamplesSplit :: Int+ , synthConfig :: SynthConfig } +data SynthConfig = SynthConfig+ { maxExprDepth :: Int+ , percentiles :: [Int]+ , enableStringOps :: Bool+ , enableCrossCols :: Bool+ , enableArithOps :: Bool+ }++defaultSynthConfig :: SynthConfig+defaultSynthConfig =+ SynthConfig+ { maxExprDepth = 2+ , percentiles = [0, 10 .. 100]+ , enableStringOps = True+ , enableCrossCols = True+ , enableArithOps = True+ }++defaultTreeConfig :: TreeConfig+defaultTreeConfig =+ TreeConfig+ { maxTreeDepth = 10+ , minSamplesSplit = 5+ , synthConfig = defaultSynthConfig+ }+ fitDecisionTree :: forall a. (Columnable a) =>@@ -43,9 +77,11 @@ fitDecisionTree cfg (Col target) df = buildTree @a cfg- (maxDepth cfg)+ (maxTreeDepth cfg) target- (generateConditions (exclude [target] df))+ ( numericConditions (synthConfig cfg) (exclude [target] df)+ ++ generateConditionsOld (synthConfig cfg) (exclude [target] df)+ ) df fitDecisionTree _ expr _ = error $ "Cannot create tree for compound expression: " ++ show expr @@ -87,6 +123,7 @@ else case (t, f) of -- Nested simplification: `if C1 then (if C1 then X else Y) else Z` -- becomes: if C1 then X else Z`+ -- Generalize this with hegg later. (If condInner tInner fInner, _) | cond == condInner -> If cond tInner f (_, If condInner tInner fInner) | cond == condInner -> If cond t fInner _ -> If cond t f@@ -94,8 +131,75 @@ pruneTree (BinaryOp name op l r) = BinaryOp name op (pruneTree l) (pruneTree r) pruneTree e = e -generateConditions :: DataFrame -> [Expr Bool]-generateConditions df =+type CondGen = SynthConfig -> DataFrame -> [Expr Bool]++numericConditions :: CondGen+numericConditions = generateNumericConds++generateNumericConds ::+ SynthConfig -> DataFrame -> [Expr Bool]+generateNumericConds cfg df = do+ expr <- numericExprsWithTerms cfg df+ let thresholds = map (\p -> percentile p expr df) (percentiles cfg)+ threshold <- thresholds+ [ expr .<= F.lit threshold+ , expr .>= F.lit threshold+ , expr .< F.lit threshold+ , expr .> F.lit threshold+ ]++numericExprsWithTerms ::+ SynthConfig -> DataFrame -> [Expr Double]+numericExprsWithTerms cfg df =+ concatMap (numericExprs cfg df [] 0) [0 .. maxExprDepth cfg]++numericCols :: DataFrame -> [Expr Double]+numericCols df = concatMap extract (columnNames df)+ where+ extract col = case unsafeGetColumn col df of+ UnboxedColumn (_ :: VU.Vector b) ->+ case testEquality (typeRep @b) (typeRep @Double) of+ Just Refl -> [Col col]+ Nothing -> case sIntegral @b of+ STrue -> [F.toDouble (Col @b col)]+ SFalse -> []+ _ -> []++numericExprs ::+ SynthConfig -> DataFrame -> [Expr Double] -> Int -> Int -> [Expr Double]+numericExprs cfg df prevExprs depth maxDepth+ | depth == 0 = baseExprs ++ numericExprs cfg df baseExprs (depth + 1) maxDepth+ | depth >= maxDepth = []+ | otherwise =+ combinedExprs ++ numericExprs cfg df combinedExprs (depth + 1) maxDepth+ where+ baseExprs = numericCols df++ combinedExprs+ | not (enableArithOps cfg) = []+ | otherwise = do+ e1 <- prevExprs+ e2 <- baseExprs+ guard (e1 /= e2)+ [e1 + e2, e1 - e2, e1 * e2, F.ifThenElse (e2 .>= 0) (e1 / e2) 0]++boolExprs ::+ DataFrame -> [Expr Bool] -> [Expr Bool] -> Int -> Int -> [Expr Bool]+boolExprs df baseExprs prevExprs depth maxDepth+ | depth == 0 =+ baseExprs ++ boolExprs df baseExprs prevExprs (depth + 1) maxDepth+ | depth >= maxDepth = []+ | otherwise =+ combinedExprs ++ boolExprs df baseExprs combinedExprs (depth + 1) maxDepth+ where+ combinedExprs = do+ e1 <- prevExprs+ e2 <- baseExprs+ guard (e1 /= e2)+ [F.and e1 e2, F.or e1 e2]++generateConditionsOld :: SynthConfig -> DataFrame -> [Expr Bool]+generateConditionsOld cfg df = let genConds :: T.Text -> [Expr Bool] genConds colName = case unsafeGetColumn colName df of@@ -103,32 +207,27 @@ let percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99] in- map (Col @a colName .<=) percentiles- ++ map (Col @a colName .==) percentiles+ map (Col @a colName .==) percentiles (OptionalColumn (col :: V.Vector a)) -> let percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99] in- map (Col @a colName .<=) percentiles- ++ map (Col @a colName .==) percentiles- (UnboxedColumn (col :: VU.Vector a)) ->- let- percentiles = map (Lit . (`percentileOrd'` VU.convert col)) [1, 25, 75, 99]- in- map (Col @a colName .<=) percentiles- ++ map (Col @a colName .==) percentiles+ map (Col @a colName .==) percentiles+ (UnboxedColumn (col :: VU.Vector a)) -> [] columnConds = concatMap colConds [(l, r) | l <- columnNames df, r <- columnNames df] where colConds (!l, !r) = case (unsafeGetColumn l df, unsafeGetColumn r df) of (BoxedColumn (col1 :: V.Vector a), BoxedColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of Nothing -> [] Just Refl -> [Col @a l .== Col @a r]- (UnboxedColumn (col1 :: VU.Vector a), UnboxedColumn (col2 :: VU.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> []- Just Refl -> [Col @a l .<= Col @a r, Col @a l .== Col @a r]- (OptionalColumn (col1 :: V.Vector a), OptionalColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> []- Just Refl -> [Col @a l .<= Col @a r, Col @a l .== Col @a r]+ (UnboxedColumn (col1 :: VU.Vector a), UnboxedColumn (col2 :: VU.Vector b)) -> []+ ( OptionalColumn (col1 :: V.Vector (Maybe a))+ , OptionalColumn (col2 :: V.Vector (Maybe b))+ ) -> case testEquality (typeRep @a) (typeRep @b) of+ Nothing -> []+ Just Refl -> case testEquality (typeRep @a) (typeRep @T.Text) of+ Nothing -> [Col @(Maybe a) l .<= Col r, Col @(Maybe a) l .== Col r]+ Just Refl -> [Col @(Maybe a) l .== Col r] _ -> [] in concatMap genConds (columnNames df) ++ columnConds@@ -142,6 +241,8 @@ T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool) findBestSplit target conds df = let+ minLeafSize = 1+ lambda = 0.05 initialImpurity = calculateGini @a target df evalGain cond = let (t, f) = partitionDataFrame cond df@@ -151,13 +252,28 @@ newImpurity = (weightT * calculateGini @a target t) + (weightF * calculateGini @a target f)- in initialImpurity - newImpurity+ in ( (initialImpurity - newImpurity) - lambda * fromIntegral (eSize cond)+ , negate (eSize cond)+ ) - validConds = filter (\c -> nRows (filterWhere c df) > 0) conds+ validConds =+ filter+ ( \c ->+ let+ (t, f) = partitionDataFrame c df+ in+ nRows t >= minLeafSize && nRows f >= minLeafSize+ )+ (nubOrd conds)+ sortedConditions = take 10 (sortBy (flip compare `on` evalGain) validConds) in if null validConds then Nothing- else Just $ maximumBy (compare `on` evalGain) validConds+ else+ Just $+ maximumBy+ (compare `on` evalGain)+ (boolExprs df sortedConditions sortedConditions 0 3) calculateGini :: forall a.@@ -166,7 +282,8 @@ calculateGini target df = let n = fromIntegral $ nRows df counts = getCounts @a target df- probs = map (\c -> fromIntegral c / n) (M.elems counts)+ numClasses = fromIntegral $ M.size counts+ probs = map (\c -> (fromIntegral c + 1) / (n + numClasses)) (M.elems counts) in if n == 0 then 0 else 1 - sum (map (^ 2) probs) majorityValue ::
src/DataFrame/Functions.hs view
@@ -33,7 +33,7 @@ import Data.Functor import qualified Data.List as L import qualified Data.Map as M-import Data.Maybe (catMaybes, fromMaybe, isJust, listToMaybe)+import qualified Data.Maybe as Maybe import qualified Data.Text as T import Data.Time import qualified Data.Vector as V@@ -107,19 +107,13 @@ -- TODO: Generalize this pattern for other equality functions. (.>) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool-(.>) (Lit l) (Lit r) = Lit (l > r)-(.>) (Lit l) expr = UnaryOp ("gt " <> T.pack (show l)) (l >) expr-(.>) expr (Lit r) = UnaryOp ("leq " <> T.pack (show r)) (r <=) expr-(.>) l r = BinaryOp "gt" (>) l r+(.>) = BinaryOp "gt" (>) gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool gt = (.>) (.<=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-(.<=) (Lit l) (Lit r) = Lit (l <= r)-(.<=) (Lit l) expr = UnaryOp ("leq " <> T.pack (show l)) (l <=) expr-(.<=) expr (Lit r) = UnaryOp ("gt " <> T.pack (show r)) (r >) expr-(.<=) l r = BinaryOp "leq" (<=) l r+(.<=) = BinaryOp "leq" (<=) leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool leq = (.<=)@@ -170,12 +164,18 @@ sum :: forall a. (Columnable a, Num a) => Expr a -> Expr a sum expr = AggReduce expr "sum" (+)+{-# SPECIALIZE DataFrame.Functions.sum :: Expr Double -> Expr Double #-}+{-# SPECIALIZE DataFrame.Functions.sum :: Expr Int -> Expr Int #-}+{-# INLINEABLE DataFrame.Functions.sum #-} sumMaybe :: forall a. (Columnable a, Num a) => Expr (Maybe a) -> Expr a-sumMaybe expr = AggVector expr "sumMaybe" (P.sum . catMaybes . V.toList)+sumMaybe expr = AggVector expr "sumMaybe" (P.sum . Maybe.catMaybes . V.toList) mean :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double mean expr = AggNumericVector expr "mean" mean'+{-# SPECIALIZE DataFrame.Functions.mean :: Expr Double -> Expr Double #-}+{-# SPECIALIZE DataFrame.Functions.mean :: Expr Int -> Expr Double #-}+{-# INLINEABLE DataFrame.Functions.mean #-} meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double meanMaybe expr = AggVector expr "meanMaybe" (mean' . optionalToDoubleVector)@@ -193,7 +193,7 @@ optionalToDoubleVector = VU.fromList . V.foldl'- (\acc e -> if isJust e then realToFrac (fromMaybe 0 e) : acc else acc)+ (\acc e -> if Maybe.isJust e then realToFrac (Maybe.fromMaybe 0 e) : acc else acc) [] percentile :: Int -> Expr Double -> Expr Double@@ -232,6 +232,21 @@ (Columnable a, Columnable b) => Expr b -> a -> (a -> b -> a) -> Expr a reduce expr = AggFold expr "foldUdf" +toMaybe :: (Columnable a) => Expr a -> Expr (Maybe a)+toMaybe = UnaryOp "toMaybe" Just++fromMaybe :: (Columnable a) => a -> Expr (Maybe a) -> Expr a+fromMaybe d = UnaryOp ("fromMaybe " <> T.pack (show d)) (Maybe.fromMaybe d)++isJust :: (Columnable a) => Expr (Maybe a) -> Expr Bool+isJust = UnaryOp "isJust" Maybe.isJust++isNothing :: (Columnable a) => Expr (Maybe a) -> Expr Bool+isNothing = UnaryOp "isNothing" Maybe.isNothing++fromJust :: (Columnable a) => Expr (Maybe a) -> Expr a+fromJust = UnaryOp "fromJust" Maybe.fromJust+ whenPresent :: forall a b. (Columnable a, Columnable b) => (a -> b) -> Expr (Maybe a) -> Expr (Maybe b)@@ -248,17 +263,26 @@ (Columnable a, Columnable b) => [(a, b)] -> Expr a -> Expr (Maybe b) recode mapping = UnaryOp (T.pack ("recode " ++ show mapping)) (`lookup` mapping) +recodeWithCondition ::+ forall a b.+ (Columnable a, Columnable b) =>+ Expr b -> [(Expr a -> Expr Bool, b)] -> Expr a -> Expr b+recodeWithCondition fallback [] value = fallback+recodeWithCondition fallback ((cond, value) : rest) expr = ifThenElse (cond expr) (lit value) (recodeWithCondition fallback rest expr)+ recodeWithDefault :: forall a b. (Columnable a, Columnable b) => b -> [(a, b)] -> Expr a -> Expr b recodeWithDefault d mapping =- UnaryOp (T.pack ("recode " ++ show mapping)) (fromMaybe d . (`lookup` mapping))+ UnaryOp+ (T.pack ("recode " ++ show mapping))+ (Maybe.fromMaybe d . (`lookup` mapping)) firstOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)-firstOrNothing = lift listToMaybe+firstOrNothing = lift Maybe.listToMaybe lastOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)-lastOrNothing = lift (listToMaybe . reverse)+lastOrNothing = lift (Maybe.listToMaybe . reverse) splitOn :: T.Text -> Expr T.Text -> Expr [T.Text] splitOn delim = lift (T.splitOn delim)
src/DataFrame/Internal/Column.hs view
@@ -1023,18 +1023,24 @@ BoxedColumn $ fmap Left left <> fmap Right (VG.convert right) concatColumnsEither (UnboxedColumn left) (BoxedColumn right) = BoxedColumn $ fmap Left (VG.convert left) <> fmap Right right-concatColumnsEither (OptionalColumn left) (BoxedColumn right) =- OptionalColumn $- fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right)-concatColumnsEither (BoxedColumn left) (OptionalColumn right) =- OptionalColumn $- fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right-concatColumnsEither (OptionalColumn left) (UnboxedColumn right) =- OptionalColumn $- fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right)-concatColumnsEither (UnboxedColumn left) (OptionalColumn right) =- OptionalColumn $- fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right+concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (BoxedColumn (right :: VB.Vector b)) =+ case testEquality (typeRep @a) (typeRep @b) of+ Just Refl -> OptionalColumn $ left <> fmap Just right+ Nothing -> OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) right+concatColumnsEither (BoxedColumn (left :: VB.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) =+ case testEquality (typeRep @a) (typeRep @b) of+ Just Refl -> OptionalColumn $ fmap Just left <> right+ Nothing -> OptionalColumn $ fmap (Just . Left) left <> fmap (fmap Right) right+concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (UnboxedColumn (right :: VU.Vector b)) =+ case testEquality (typeRep @a) (typeRep @b) of+ Just Refl -> OptionalColumn $ left <> fmap Just (VG.convert right)+ Nothing ->+ OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right)+concatColumnsEither (UnboxedColumn (left :: VU.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) =+ case testEquality (typeRep @a) (typeRep @b) of+ Just Refl -> OptionalColumn $ fmap Just (VG.convert left) <> right+ Nothing ->+ OptionalColumn $ fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right {- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified.
src/DataFrame/Internal/Statistics.hs view
@@ -18,13 +18,28 @@ mean' samp | VU.null samp = throw $ EmptyDataSetException "mean" | otherwise = rtf (VU.sum samp) / fromIntegral (VU.length samp)-{-# INLINE mean' #-}+{-# INLINE [0] mean' #-} meanDouble' :: VU.Vector Double -> Double meanDouble' samp | VU.null samp = throw $ EmptyDataSetException "mean" | otherwise = VU.sum samp / fromIntegral (VU.length samp) {-# INLINE meanDouble' #-}++meanInt' :: VU.Vector Int -> Double+meanInt' samp+ | VU.null samp = throw $ EmptyDataSetException "mean"+ | otherwise = fromIntegral (VU.sum samp) / fromIntegral (VU.length samp)+{-# INLINE meanInt' #-}++{-# RULES+"mean'/Double" [1] forall (xs :: VU.Vector Double).+ mean' xs =+ meanDouble' xs+"mean'/Int" [1] forall (xs :: VU.Vector Int).+ mean' xs =+ meanInt' xs+ #-} median' :: (Real a, VU.Unbox a) => VU.Vector a -> Double median' samp
src/DataFrame/Operations/Statistics.hs view
@@ -119,6 +119,44 @@ Left e -> throw e Right xs -> median' xs +-- | Calculates the median of a given column (containing optional values) as a standalone value.+medianMaybe ::+ forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double+medianMaybe (Col name) df =+ (median' . optionalToDoubleVector)+ (either throw id (columnAsVector (Col @(Maybe a) name) df))+medianMaybe expr df = case interpret @(Maybe a) df expr of+ Left e -> throw e+ Right (TColumn col) -> case toVector @(Maybe a) col of+ Left e -> throw e+ Right xs -> (median' . optionalToDoubleVector) xs++-- | Calculates the nth percentile of a given column as a standalone value.+percentile ::+ forall a.+ (Columnable a, Real a, VU.Unbox a) => Int -> Expr a -> DataFrame -> Double+percentile n (Col name) df = case columnAsUnboxedVector (Col @a name) df of+ Right xs -> percentile' n xs+ Left e -> throw e+percentile n expr df = case interpret df expr of+ Left e -> throw e+ Right (TColumn col) -> case toUnboxedVector @a col of+ Left e -> throw e+ Right xs -> percentile' n xs++-- | Calculates the nth percentile of a given column as a standalone value.+genericPercentile ::+ forall a.+ (Columnable a, Ord a) => Int -> Expr a -> DataFrame -> a+genericPercentile n (Col name) df = case columnAsVector (Col @a name) df of+ Right xs -> percentileOrd' n xs+ Left e -> throw e+genericPercentile n expr df = case interpret df expr of+ Left e -> throw e+ Right (TColumn col) -> case toVector @a col of+ Left e -> throw e+ Right xs -> percentileOrd' n xs+ -- | Calculates the standard deviation of a given column as a standalone value. standardDeviation :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double
tests/Functions.hs view
@@ -6,11 +6,9 @@ import qualified DataFrame as D import DataFrame.Functions ( sanitize,- (.>), ) import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI-import DataFrame.Internal.Expression import Test.HUnit -- Test cases for the sanitize function@@ -81,48 +79,8 @@ (D.derive "sum" (F.sum (F.col @Int "A")) df) ) -testGtTwoLitsRewriteAsLit :: Test-testGtTwoLitsRewriteAsLit =- TestCase- ( assertEqual- "Two lits rewrite as one"- (Lit True)- (Lit (5 :: Int) .> Lit 4)- )--testGtLeftLitRewriteAsUnary :: Test-testGtLeftLitRewriteAsUnary =- TestCase- ( assertEqual- "Two lits rewrite as single literal"- (UnaryOp "gt 5" ((5 :: Int) >) (Col "x"))- (Lit (5 :: Int) .> Col "x")- )--testGtRightLitRewriteAsUnary :: Test-testGtRightLitRewriteAsUnary =- TestCase- ( assertEqual- "Two lits rewrite as unary op"- (UnaryOp "gt 5" ((5 :: Int) >) (Col "x"))- (Lit (5 :: Int) .> Col "x")- )--testGtNoLitWriteAsBinary :: Test-testGtNoLitWriteAsBinary =- TestCase- ( assertEqual- "Two lits rewrite as unary op"- (BinaryOp "gt" (>) (Col @Int "x") (Col "y"))- (Col @Int "x" .> Col "y")- )- tests :: [Test] tests = [ TestLabel "sanitizeIdentifiers" sanitizeIdentifiers , TestLabel "testSum" testSum- , TestLabel "testGtTwoLitRewrite" testGtTwoLitsRewriteAsLit- , TestLabel "testGtLeftLit" testGtLeftLitRewriteAsUnary- , TestLabel "testGtRightLit" testGtRightLitRewriteAsUnary- , TestLabel "testGtNoLit" testGtNoLitWriteAsBinary ]