dataframe 0.4.0.0 → 0.4.0.2
raw patch · 15 files changed
+971/−686 lines, 15 files
Files
- CHANGELOG.md +11/−0
- app/Benchmark.hs +32/−26
- dataframe.cabal +3/−2
- src/DataFrame.hs +1/−0
- src/DataFrame/DecisionTree.hs +192/−0
- src/DataFrame/Functions.hs +6/−3
- src/DataFrame/IO/CSV.hs +1/−1
- src/DataFrame/Internal/Column.hs +94/−4
- src/DataFrame/Internal/DataFrame.hs +0/−161
- src/DataFrame/Internal/Expression.hs +414/−345
- src/DataFrame/Internal/Row.hs +5/−3
- src/DataFrame/Internal/Statistics.hs +79/−31
- src/DataFrame/Operations/Core.hs +110/−96
- src/DataFrame/Operations/Statistics.hs +11/−9
- src/DataFrame/Synthesis.hs +12/−5
CHANGELOG.md view
@@ -1,5 +1,16 @@ # Revision history for dataframe +## 0.4.0.2+* Improved performance for folds and reductions.+* Improve standalone mean and correlation functions.+* Remove buggy boxedness check in aggregations.+* CSV files shouldn't have spaces in headers.+* Small decision tree implementation (experimental).++## 0.4.0.1+* Fuse literals in binary expressions and conditionals: we can now express computations like: `df |> D.groupBy [F.name ocean_proximity] |> D.aggregate ["rand" .= F.sum (F.ifThenElse (ocean_proximity .== "ISLAND") 1 0)]`.+* Unary aggregations do not mistakenly boxed unboxed instances.+ ## 0.4.0.0 * `readSeparated` no longer takes the separator as an argument. This is not placed into readOptions. * Some improvements to the synthesis demo
app/Benchmark.hs view
@@ -1,5 +1,6 @@ {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE TypeApplications #-} import Data.Time@@ -8,31 +9,36 @@ import qualified DataFrame.Functions as F import System.Random.Stateful +import Data.Text (Text)+import DataFrame ((|>))+import DataFrame.DecisionTree+import DataFrame.Functions ((.=))++$(F.declareColumnsFromCsvFile "../../Downloads/playground-series-s5e11/train.csv")+ main :: IO () main = do- let n = 100_000_000- g <- newIOGenM =<< newStdGen- let range = (0 :: Double, 1 :: Double)- startGeneration <- getCurrentTime- ns <- VU.replicateM n (uniformRM range g)- xs <- VU.replicateM n (uniformRM range g)- ys <- VU.replicateM n (uniformRM range g)- let df = D.fromUnnamedColumns (map D.fromUnboxedVector [ns, xs, ys])- print df- endGeneration <- getCurrentTime- let generationTime = diffUTCTime endGeneration startGeneration- putStrLn $ "Data generation Time: " ++ show generationTime- startCalculation <- getCurrentTime- print $ D.mean (F.col @Double "0") df- print $ D.variance (F.col @Double "1") df- print $ D.correlation "1" "2" df- endCalculation <- getCurrentTime- let calculationTime = diffUTCTime endCalculation startCalculation- putStrLn $ "Calculation Time: " ++ show calculationTime- startFilter <- getCurrentTime- print $ D.filter (F.col @Double "0") (> 0.971) df D.|> D.take 10- endFilter <- getCurrentTime- let filterTime = diffUTCTime endFilter startFilter- putStrLn $ "Filter Time: " ++ show filterTime- let totalTime = diffUTCTime endFilter startGeneration- putStrLn $ "Total Time: " ++ show totalTime+ train <- D.readCsv "../../Downloads/playground-series-s5e11/train.csv"+ -- Create a new symbol for loan paid back since we are changing the type.+ let (loanPaidBack, train') =+ train+ |> D.deriveWithExpr+ (F.name loan_paid_back)+ (F.lift (round @Double @Int) loan_paid_back)++ let model = fitDecisionTree (TreeConfig 15 2) loanPaidBack (train' |> D.exclude ["id"])+ let trainPred = D.derive "prediction" model train'+ print $+ trainPred+ |> D.groupBy [F.name loanPaidBack, "prediction"]+ |> D.aggregate ["count" .= F.count loanPaidBack]+ |> D.sortBy [D.Desc "prediction", D.Desc (F.name loanPaidBack)]++ test <- D.readCsv "../../Downloads/playground-series-s5e11/test.csv"+ let withPredictions = D.derive "prediction" model test+ D.writeCsv+ "predictions.csv"+ ( withPredictions+ |> D.select ["id", "prediction"]+ |> D.rename "prediction" (F.name loan_paid_back)+ )
dataframe.cabal view
@@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe-version: 0.4.0.0+version: 0.4.0.2 synopsis: A fast, safe, and intuitive DataFrame library. @@ -84,7 +84,8 @@ DataFrame.IO.Parquet.Types, DataFrame.Lazy.IO.CSV, DataFrame.Lazy.Internal.DataFrame,- DataFrame.Monad+ DataFrame.Monad,+ DataFrame.DecisionTree build-depends: base >= 4 && <5, aeson >= 0.11.0.0 && < 3, array >= 0.5.4.0 && < 0.6,
src/DataFrame.hs view
@@ -276,6 +276,7 @@ ) import DataFrame.Internal.Expression as Expression (Expr) import DataFrame.Internal.Row as Row (+ Any, Row, fromAny, toAny,
+ src/DataFrame/DecisionTree.hs view
@@ -0,0 +1,192 @@+{-# LANGUAGE AllowAmbiguousTypes #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE GADTs #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++module DataFrame.DecisionTree where++import qualified DataFrame.Functions as F+import DataFrame.Internal.Column+import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn)+import DataFrame.Internal.Expression (Expr (..), interpret)+import DataFrame.Internal.Statistics (percentileOrd')+import DataFrame.Operations.Core (columnNames, nRows)+import DataFrame.Operations.Subset (exclude, filterWhere)++import Control.Exception (throw)+import Data.Function (on)+import Data.List (foldl', maximumBy)+import qualified Data.Map.Strict as M+import qualified Data.Text as T+import Data.Type.Equality+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import Type.Reflection (typeRep)++import DataFrame.Functions ((.<=), (.==))++data TreeConfig = TreeConfig+ { maxDepth :: Int+ , minSamplesSplit :: Int+ }++fitDecisionTree ::+ forall a.+ (Columnable a) =>+ TreeConfig ->+ Expr a ->+ DataFrame ->+ Expr a+fitDecisionTree cfg (Col target) df =+ buildTree @a+ cfg+ (maxDepth cfg)+ target+ (generateConditions (exclude [target] df))+ df+fitDecisionTree _ expr _ = error $ "Cannot create tree for compound expression: " ++ show expr++buildTree ::+ forall a.+ (Columnable a) =>+ TreeConfig ->+ Int ->+ T.Text ->+ [Expr Bool] ->+ DataFrame ->+ Expr a+buildTree cfg depth target conds df+ | depth <= 0 || nRows df <= minSamplesSplit cfg =+ Lit (majorityValue @a target df)+ | otherwise =+ case findBestSplit @a target conds df of+ Nothing -> Lit (majorityValue @a target df)+ Just bestCond ->+ let (dfTrue, dfFalse) = partitionDataFrame bestCond df+ in if nRows dfTrue == 0 || nRows dfFalse == 0+ then Lit (majorityValue @a target df)+ else+ pruneTree+ ( F.ifThenElse+ bestCond+ (buildTree @a cfg (depth - 1) target conds dfTrue)+ (buildTree @a cfg (depth - 1) target conds dfFalse)+ )++pruneTree :: forall a. (Columnable a, Eq a) => Expr a -> Expr a+pruneTree (If cond trueBranch falseBranch) =+ let+ t = pruneTree trueBranch+ f = pruneTree falseBranch+ in+ if t == f+ then t+ else case (t, f) of+ -- Nested simplification: `if C1 then (if C1 then X else Y) else Z`+ -- becomes: if C1 then X else Z`+ (If condInner tInner fInner, _) | cond == condInner -> If cond tInner f+ (_, If condInner tInner fInner) | cond == condInner -> If cond t fInner+ _ -> If cond t f+pruneTree (UnaryOp name op e) = UnaryOp name op (pruneTree e)+pruneTree (BinaryOp name op l r) = BinaryOp name op (pruneTree l) (pruneTree r)+pruneTree e = e++generateConditions :: DataFrame -> [Expr Bool]+generateConditions df =+ let+ genConds :: T.Text -> [Expr Bool]+ genConds colName = case unsafeGetColumn colName df of+ (BoxedColumn (col :: V.Vector a)) ->+ let+ percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99]+ in+ map (Col @a colName .<=) percentiles+ ++ map (Col @a colName .==) percentiles+ (OptionalColumn (col :: V.Vector a)) ->+ let+ percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99]+ in+ map (Col @a colName .<=) percentiles+ ++ map (Col @a colName .==) percentiles+ (UnboxedColumn (col :: VU.Vector a)) ->+ let+ percentiles = map (Lit . (`percentileOrd'` VU.convert col)) [1, 25, 75, 99]+ in+ map (Col @a colName .<=) percentiles+ ++ map (Col @a colName .==) percentiles+ columnConds = concatMap colConds [(l, r) | l <- columnNames df, r <- columnNames df]+ where+ colConds (!l, !r) = case (unsafeGetColumn l df, unsafeGetColumn r df) of+ (BoxedColumn (col1 :: V.Vector a), BoxedColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of+ Nothing -> []+ Just Refl -> [Col @a l .== Col @a r]+ (UnboxedColumn (col1 :: VU.Vector a), UnboxedColumn (col2 :: VU.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of+ Nothing -> []+ Just Refl -> [Col @a l .<= Col @a r, Col @a l .== Col @a r]+ (OptionalColumn (col1 :: V.Vector a), OptionalColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of+ Nothing -> []+ Just Refl -> [Col @a l .<= Col @a r, Col @a l .== Col @a r]+ _ -> []+ in+ concatMap genConds (columnNames df) ++ columnConds++partitionDataFrame :: Expr Bool -> DataFrame -> (DataFrame, DataFrame)+partitionDataFrame cond df = (filterWhere cond df, filterWhere (F.not cond) df)++findBestSplit ::+ forall a.+ (Columnable a) =>+ T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool)+findBestSplit target conds df =+ let+ initialImpurity = calculateGini @a target df+ evalGain cond =+ let (t, f) = partitionDataFrame cond df+ n = fromIntegral @Int @Double (nRows df)+ weightT = fromIntegral @Int @Double (nRows t) / n+ weightF = fromIntegral @Int @Double (nRows f) / n+ newImpurity =+ (weightT * calculateGini @a target t)+ + (weightF * calculateGini @a target f)+ in initialImpurity - newImpurity++ validConds = filter (\c -> nRows (filterWhere c df) > 0) conds+ in+ if null validConds+ then Nothing+ else Just $ maximumBy (compare `on` evalGain) validConds++calculateGini ::+ forall a.+ (Columnable a) =>+ T.Text -> DataFrame -> Double+calculateGini target df =+ let n = fromIntegral $ nRows df+ counts = getCounts @a target df+ probs = map (\c -> fromIntegral c / n) (M.elems counts)+ in if n == 0 then 0 else 1 - sum (map (^ 2) probs)++majorityValue ::+ forall a.+ (Columnable a) =>+ T.Text -> DataFrame -> a+majorityValue target df =+ let counts = getCounts @a target df+ in if M.null counts+ then error "Empty DataFrame in leaf"+ else fst $ maximumBy (compare `on` snd) (M.toList counts)++getCounts ::+ forall a.+ (Columnable a) =>+ T.Text -> DataFrame -> M.Map a Int+getCounts target df =+ case interpret @a df (Col target) of+ Left e -> throw e+ Right (TColumn col) ->+ case toVector @a col of+ Left e -> throw e+ Right vals -> foldl' (\acc x -> M.insertWith (+) x 1 acc) M.empty (V.toList vals)
src/DataFrame/Functions.hs view
@@ -116,10 +116,13 @@ gt = (.>) (.<=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-(.<=) = BinaryOp "leq" (<=)+(.<=) (Lit l) (Lit r) = Lit (l <= r)+(.<=) (Lit l) expr = UnaryOp ("leq " <> T.pack (show l)) (l <=) expr+(.<=) expr (Lit r) = UnaryOp ("gt " <> T.pack (show r)) (r >) expr+(.<=) l r = BinaryOp "leq" (<=) l r leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool-leq = BinaryOp "leq" (<=)+leq = (.<=) (.>=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool (.>=) = BinaryOp "geq" (>=)@@ -166,7 +169,7 @@ maximum expr = AggReduce expr "maximum" Prelude.max sum :: forall a. (Columnable a, Num a) => Expr a -> Expr a-sum expr = AggFold expr "sum" 0 (+)+sum expr = AggReduce expr "sum" (+) sumMaybe :: forall a. (Columnable a, Num a) => Expr (Maybe a) -> Expr a sumMaybe expr = AggVector expr "sumMaybe" (P.sum . catMaybes . V.toList)
src/DataFrame/IO/CSV.hs view
@@ -379,7 +379,7 @@ writeSeparated c filepath df = withFile filepath WriteMode $ \handle -> do let (rows, _) = dataframeDimensions df let headers = map fst (L.sortBy (compare `on` snd) (M.toList (columnIndices df)))- TIO.hPutStrLn handle (T.intercalate ", " headers)+ TIO.hPutStrLn handle (T.intercalate "," headers) forM_ [0 .. (rows - 1)] $ \i -> do let row = getRowAsText df i TIO.hPutStrLn handle (T.intercalate "," row)
src/DataFrame/Internal/Column.hs view
@@ -280,7 +280,7 @@ | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> Right $ case sUnbox @c of STrue -> UnboxedColumn (VU.map f col)- SFalse -> fromVector @c (VB.map f (VB.convert col))+ SFalse -> fromVector @c (VB.generate (VU.length col) (f . VU.unsafeIndex col)) | otherwise -> Left $ TypeMismatchException@@ -291,6 +291,10 @@ , errorColumnName = Nothing } )+{-# SPECIALIZE mapColumn ::+ (Double -> Double) -> Column -> Either DataFrameException Column+ #-}+{-# INLINEABLE mapColumn #-} -- | O(1) Gets the number of elements in the column. columnLength :: Column -> Int@@ -606,6 +610,88 @@ } ) +foldlColumn ::+ forall a b.+ (Columnable a, Columnable b) =>+ (b -> a -> b) -> b -> Column -> Either DataFrameException b+foldlColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+ Just Refl -> pure $ VG.foldl' f acc column+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @a)+ , expectedType = Right (typeRep @d)+ , callingFunctionName = Just "foldlColumn"+ , errorColumnName = Nothing+ }+ )+foldlColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+ Just Refl -> pure $ VG.foldl' f acc column+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @a)+ , expectedType = Right (typeRep @d)+ , callingFunctionName = Just "foldlColumn"+ , errorColumnName = Nothing+ }+ )+foldlColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+ Just Refl -> pure $ VG.foldl' f acc column+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @a)+ , expectedType = Right (typeRep @d)+ , callingFunctionName = Just "foldlColumn"+ , errorColumnName = Nothing+ }+ )++foldl1Column ::+ forall a.+ (Columnable a) =>+ (a -> a -> a) -> Column -> Either DataFrameException a+foldl1Column f c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+ Just Refl -> pure $ VG.foldl1' f column+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @a)+ , expectedType = Right (typeRep @d)+ , callingFunctionName = Just "foldl1Column"+ , errorColumnName = Nothing+ }+ )+foldl1Column f c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+ Just Refl -> pure $ VG.foldl1' f column+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @a)+ , expectedType = Right (typeRep @d)+ , callingFunctionName = Just "foldl1Column"+ , errorColumnName = Nothing+ }+ )+foldl1Column f c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+ Just Refl -> pure $ VG.foldl1' f column+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @a)+ , expectedType = Right (typeRep @d)+ , callingFunctionName = Just "foldl1Column"+ , errorColumnName = Nothing+ }+ )+ headColumn :: forall a. (Columnable a) => Column -> Either DataFrameException a headColumn (BoxedColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of Just Refl ->@@ -803,7 +889,7 @@ . VB.imap ( \i v -> if i `elem` map fst nulls- then Left (fromMaybe (error "") (lookup i nulls))+ then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) else Right v ) <$> VB.unsafeFreeze col@@ -824,7 +910,7 @@ (VU.length c) ( \i -> if i `elem` map fst nulls- then Left (fromMaybe (error "") (lookup i nulls))+ then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) else Right (c VU.! i) ) {-# INLINE freezeColumn' #-}@@ -1265,7 +1351,7 @@ ( MkTypeErrorContext { userType = Right (typeRep @Int) , expectedType = Right (typeRep @a)- , callingFunctionName = Just "toIntVector"+ , callingFunctionName = Just "toUnboxedVector" , errorColumnName = Nothing } )@@ -1279,3 +1365,7 @@ , errorColumnName = Nothing } )+{-# SPECIALIZE toUnboxedVector ::+ Column -> Either DataFrameException (VU.Vector Double)+ #-}+{-# INLINE toUnboxedVector #-}
src/DataFrame/Internal/DataFrame.hs view
@@ -11,7 +11,6 @@ import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Vector as V-import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU import Control.Exception (throw)@@ -162,163 +161,3 @@ -} null :: DataFrame -> Bool null df = V.null (columns df)--{- | Returns a dataframe as a two dimensional vector of floats.--Converts all columns in the dataframe to float vectors and transposes them-into a row-major matrix representation.--This is useful for handing data over into ML systems.--Returns 'Left' with an error if any column cannot be converted to floats.--}-toFloatMatrix ::- DataFrame -> Either DataFrameException (V.Vector (VU.Vector Float))-toFloatMatrix df = case V.foldl'- (\acc c -> V.snoc <$> acc <*> toFloatVector c)- (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Float)))- (columns df) of- Left e -> Left e- Right m ->- pure $- V.generate- (fst (dataframeDimensions df))- ( \i ->- foldl- (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))- VU.empty- [0 .. (V.length m - 1)]- )--{- | Returns a dataframe as a two dimensional vector of doubles.--Converts all columns in the dataframe to double vectors and transposes them-into a row-major matrix representation.--This is useful for handing data over into ML systems.--Returns 'Left' with an error if any column cannot be converted to doubles.--}-toDoubleMatrix ::- DataFrame -> Either DataFrameException (V.Vector (VU.Vector Double))-toDoubleMatrix df = case V.foldl'- (\acc c -> V.snoc <$> acc <*> toDoubleVector c)- (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Double)))- (columns df) of- Left e -> Left e- Right m ->- pure $- V.generate- (fst (dataframeDimensions df))- ( \i ->- foldl- (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))- VU.empty- [0 .. (V.length m - 1)]- )--{- | Returns a dataframe as a two dimensional vector of ints.--Converts all columns in the dataframe to int vectors and transposes them-into a row-major matrix representation.--This is useful for handing data over into ML systems.--Returns 'Left' with an error if any column cannot be converted to ints.--}-toIntMatrix :: DataFrame -> Either DataFrameException (V.Vector (VU.Vector Int))-toIntMatrix df = case V.foldl'- (\acc c -> V.snoc <$> acc <*> toIntVector c)- (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Int)))- (columns df) of- Left e -> Left e- Right m ->- pure $- V.generate- (fst (dataframeDimensions df))- ( \i ->- foldl- (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))- VU.empty- [0 .. (V.length m - 1)]- )--{- | Get a specific column as a vector.--You must specify the type via type applications.--==== __Examples__-->>> columnAsVector @Int "age" df-[25, 30, 35, ...]-->>> columnAsVector @Text "name" df-["Alice", "Bob", "Charlie", ...]--==== __Throws__--* 'error' - if the column type doesn't match the requested type--}-columnAsVector :: forall a. (Columnable a) => T.Text -> DataFrame -> V.Vector a-columnAsVector name df = case unsafeGetColumn name df of- (BoxedColumn (col :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> error "Type error"- Just Refl -> col- (OptionalColumn (col :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> error "Type error"- Just Refl -> col- (UnboxedColumn (col :: VU.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> error "Type error"- Just Refl -> VG.convert col--{- | Retrieves a column as an unboxed vector of 'Int' values.--Returns 'Left' with a 'DataFrameException' if the column cannot be converted to ints.-This may occur if the column contains non-numeric data or values outside the 'Int' range.--}-columnAsIntVector ::- T.Text -> DataFrame -> Either DataFrameException (VU.Vector Int)-columnAsIntVector name df = toIntVector (unsafeGetColumn name df)--{- | Retrieves a column as an unboxed vector of 'Double' values.--Returns 'Left' with a 'DataFrameException' if the column cannot be converted to doubles.-This may occur if the column contains non-numeric data.--}-columnAsDoubleVector ::- T.Text -> DataFrame -> Either DataFrameException (VU.Vector Double)-columnAsDoubleVector name df = toDoubleVector (unsafeGetColumn name df)--{- | Retrieves a column as an unboxed vector of 'Float' values.--Returns 'Left' with a 'DataFrameException' if the column cannot be converted to floats.-This may occur if the column contains non-numeric data.--}-columnAsFloatVector ::- T.Text -> DataFrame -> Either DataFrameException (VU.Vector Float)-columnAsFloatVector name df = toFloatVector (unsafeGetColumn name df)--columnAsUnboxedVector ::- forall a.- (Columnable a, VU.Unbox a) =>- T.Text -> DataFrame -> Either DataFrameException (VU.Vector a)-columnAsUnboxedVector name df = toUnboxedVector @a (unsafeGetColumn name df)--{- | Get a specific column as a list.--You must specify the type via type applications.--==== __Examples__-->>> columnAsList @Int "age" df-[25, 30, 35, ...]-->>> columnAsList @Text "name" df-["Alice", "Bob", "Charlie", ...]--==== __Throws__--* 'error' - if the column type doesn't match the requested type--}-columnAsList :: forall a. (Columnable a) => T.Text -> DataFrame -> [a]-columnAsList name df = V.toList (columnAsVector name df)
src/DataFrame/Internal/Expression.hs view
@@ -37,66 +37,22 @@ Col :: (Columnable a) => T.Text -> Expr a Lit :: (Columnable a) => a -> Expr a UnaryOp ::- ( Columnable a- , Columnable b- ) =>- T.Text -> -- Operation name- (b -> a) ->- Expr b ->- Expr a+ (Columnable a, Columnable b) => T.Text -> (b -> a) -> Expr b -> Expr a BinaryOp ::- ( Columnable c- , Columnable b- , Columnable a- ) =>- T.Text -> -- operation name- (c -> b -> a) ->- Expr c ->- Expr b ->- Expr a- If ::- (Columnable a) =>- Expr Bool ->- Expr a ->- Expr a ->- Expr a+ (Columnable c, Columnable b, Columnable a) =>+ T.Text -> (c -> b -> a) -> Expr c -> Expr b -> Expr a+ If :: (Columnable a) => Expr Bool -> Expr a -> Expr a -> Expr a AggVector ::- ( VG.Vector v b- , Typeable v- , Columnable a- , Columnable b- ) =>- Expr b ->- T.Text -> -- Operation name- (v b -> a) ->- Expr a- AggReduce ::- (Columnable a) =>- Expr a ->- T.Text -> -- Operation name- (a -> a -> a) ->- Expr a+ (VG.Vector v b, Typeable v, Columnable a, Columnable b) =>+ Expr b -> T.Text -> (v b -> a) -> Expr a+ AggReduce :: (Columnable a) => Expr a -> T.Text -> (a -> a -> a) -> Expr a -- TODO(mchav): Numeric reduce might be superfluous since expressions are already type checked. AggNumericVector ::- ( Columnable a- , Columnable b- , VU.Unbox a- , VU.Unbox b- , Num a- , Num b- ) =>- Expr b ->- T.Text -> -- Operation name- (VU.Vector b -> a) ->- Expr a+ (Columnable a, Columnable b, VU.Unbox a, VU.Unbox b, Num a, Num b) =>+ Expr b -> T.Text -> (VU.Vector b -> a) -> Expr a AggFold :: forall a b.- (Columnable a, Columnable b) =>- Expr b ->- T.Text -> -- Operation name- a ->- (a -> b -> a) ->- Expr a+ (Columnable a, Columnable b) => Expr b -> T.Text -> a -> (a -> b -> a) -> Expr a data UExpr where Wrap :: (Columnable a) => Expr a -> UExpr@@ -165,50 +121,13 @@ (BoxedColumn col) -> processColumn col (OptionalColumn col) -> processColumn col (UnboxedColumn col) -> processColumn col-interpret df expression@(AggReduce expr op (f :: a -> a -> a)) = case interpret @a df expr of- Left (TypeMismatchException context) ->- Left $- TypeMismatchException- ( context- { callingFunctionName = Just "interpret"- , errorColumnName = Just (show expr)- }- )- Left e -> Left e- Right (TColumn column) -> case headColumn @a column of- Left (TypeMismatchException context) ->- Left $- TypeMismatchException- ( context- { callingFunctionName = Just "interpret"- , errorColumnName = Just (show expr)- }- )- Left (EmptyDataSetException loc) -> Left (EmptyDataSetException (T.pack $ show expr))- Left e -> Left e- Right h -> case ifoldlColumn (\acc _ v -> f acc v) h column of- Left (TypeMismatchException context) ->- Left $- TypeMismatchException- ( context- { callingFunctionName = Just "interpret"- , errorColumnName = Just (show expression)- }- )- Left e -> Left e- Right value ->- pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value-interpret df expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) = case interpret @b df expr of- Left (TypeMismatchException context) ->- Left $- TypeMismatchException- ( context- { callingFunctionName = Just "interpret"- , errorColumnName = Just (show expr)- }- )- Left e -> Left e- Right (TColumn column) -> case column of+interpret df expression@(AggReduce expr op (f :: a -> a -> a)) = first (handleInterpretException (show expr)) $ do+ (TColumn column) <- interpret @a df expr+ value <- foldl1Column f column+ pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value+interpret df expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) = first (handleInterpretException (show expression)) $ do+ (TColumn column) <- interpret @b df expr+ case column of (UnboxedColumn (v :: VU.Vector d)) -> case testEquality (typeRep @d) (typeRep @b) of Just Refl -> pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f v)@@ -216,222 +135,21 @@ Left $ TypeMismatchException ( MkTypeErrorContext- { userType = Right (typeRep @b)- , expectedType = Right (typeRep @d)- , callingFunctionName = Just "interpret"- , errorColumnName = Just (show expression)- }- )- (BoxedColumn (v :: V.Vector d)) -> case testEquality (typeRep @d) (typeRep @Integer) of- Just Refl ->- Right $- TColumn $- fromVector $- V.replicate- (fst $ dataframeDimensions df)- (f (VU.convert $ V.map fromInteger v))- Nothing ->- Left $- TypeMismatchException- ( MkTypeErrorContext- { userType = Right (typeRep @Integer)- , expectedType = Right (typeRep @d)- , callingFunctionName = Just "interpret"- , errorColumnName = Just (show expression)- }- )- (OptionalColumn (v :: V.Vector (Maybe d))) -> case sNumeric @d of- STrue -> case testEquality (typeRep @d) (typeRep @b) of- Nothing ->- Left $- TypeMismatchException- ( MkTypeErrorContext- { userType = Right (typeRep @b)- , expectedType = Right (typeRep @d)- , callingFunctionName = Just "interpret"- , errorColumnName = Just (show expression)- }- )- Just Refl ->- pure $- TColumn $- fromVector $- V.replicate- (fst $ dataframeDimensions df)- (f (VU.convert $ V.map (fromMaybe 0) $ V.filter isJust v))- SFalse ->- Left $- TypeMismatchException- ( MkTypeErrorContext- { userType = Right (typeRep @d)- , expectedType = Right (typeRep @b)- , callingFunctionName = Just "interpret"- , errorColumnName = Just (show expression)- }+ (Right (typeRep @b))+ (Right (typeRep @d))+ (Just "interpret")+ (Just (show expression)) )-interpret df expression@(AggFold expr op start (f :: (a -> b -> a))) = case interpret @b df expr of- Left (TypeMismatchException context) ->- Left $- TypeMismatchException- ( context- { callingFunctionName = Just "interpret"- , errorColumnName = Just (show expr)- }- )- Left e -> Left e- Right (TColumn column) -> case ifoldlColumn (\acc _ v -> f acc v) start column of- Left (TypeMismatchException context) ->- Left $- TypeMismatchException- ( context- { callingFunctionName = Just "interpret"- , errorColumnName = Just (show expression)- }- )- Left e -> Left e- Right value ->- pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value+ _ -> error "Trying to apply numeric computation to non-numeric column"+interpret df expression@(AggFold expr op start (f :: (a -> b -> a))) = first (handleInterpretException (show expression)) $ do+ (TColumn column) <- interpret @b df expr+ value <- foldlColumn f start column+ pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value data AggregationResult a = UnAggregated Column | Aggregated (TypedColumn a) -mkUnaggregatedColumnBoxed ::- forall a.- (Columnable a) =>- V.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (V.Vector a)-mkUnaggregatedColumnBoxed col os indices =- let- sorted = V.unsafeBackpermute col (V.convert indices)- n i = os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)- start i = os `VG.unsafeIndex` i- in- V.generate- (VU.length os - 1)- ( \i ->- V.unsafeSlice (start i) (n i) sorted- )--mkUnaggregatedColumnUnboxed ::- forall a.- (Columnable a, VU.Unbox a) =>- VU.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (VU.Vector a)-mkUnaggregatedColumnUnboxed col os indices =- let- sorted = VU.unsafeBackpermute col indices- n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i)- start i = os `VG.unsafeIndex` i- in- V.generate- (VU.length os - 1)- ( \i ->- VU.unsafeSlice (start i) (n i) sorted- )--mkAggregatedColumnUnboxed ::- forall a b.- (Columnable a, VU.Unbox a, Columnable b, VU.Unbox b) =>- VU.Vector a ->- VU.Vector Int ->- VU.Vector Int ->- (VU.Vector a -> b) ->- VU.Vector b-mkAggregatedColumnUnboxed col os indices f =- let- sorted = VU.unsafeBackpermute col indices- n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i)- start i = os `VG.unsafeIndex` i- in- VU.generate- (VU.length os - 1)- ( \i ->- f (VU.unsafeSlice (start i) (n i) sorted)- )--mkReducedColumnUnboxed ::- forall a.- (VU.Unbox a) =>- VU.Vector a ->- VU.Vector Int ->- VU.Vector Int ->- (a -> a -> a) ->- VU.Vector a-mkReducedColumnUnboxed col os indices f = runST $ do- let len = VU.length os - 1- mvec <- VUM.unsafeNew len-- let loopOut i- | i == len = return ()- | otherwise = do- let start = os `VU.unsafeIndex` i- let end = os `VU.unsafeIndex` (i + 1)- let initVal = col `VU.unsafeIndex` (indices `VU.unsafeIndex` start)-- let loopIn !acc idx- | idx == end = acc- | otherwise =- let val = col `VU.unsafeIndex` (indices `VU.unsafeIndex` idx)- in loopIn (f acc val) (idx + 1)- let !finalVal = loopIn initVal (start + 1)- VUM.unsafeWrite mvec i finalVal- loopOut (i + 1)-- loopOut 0- VU.unsafeFreeze mvec-{-# INLINE mkReducedColumnUnboxed #-}--mkReducedColumnBoxed ::- V.Vector a ->- VU.Vector Int ->- VU.Vector Int ->- (a -> a -> a) ->- V.Vector a-mkReducedColumnBoxed col os indices f = runST $ do- let len = VU.length os - 1- mvec <- VM.unsafeNew len-- let loopOut i- | i == len = return ()- | otherwise = do- let start = os `VU.unsafeIndex` i- let end = os `VU.unsafeIndex` (i + 1)- let initVal = col `V.unsafeIndex` (indices `VU.unsafeIndex` start)-- let loopIn !acc idx- | idx == end = acc- | otherwise =- let val = col `V.unsafeIndex` (indices `VU.unsafeIndex` idx)- in loopIn (f acc val) (idx + 1)- let !finalVal = loopIn initVal (start + 1)- VM.unsafeWrite mvec i finalVal- loopOut (i + 1)-- loopOut 0- V.unsafeFreeze mvec-{-# INLINE mkReducedColumnBoxed #-}--nestedTypeException ::- forall a b. (Typeable a, Typeable b) => String -> DataFrameException-nestedTypeException expression = case typeRep @a of- App t1 t2 ->- TypeMismatchException- ( MkTypeErrorContext- { userType = Left (show (typeRep @b)) :: Either String (TypeRep ())- , expectedType = Left (show (typeRep @a)) :: Either String (TypeRep ())- , callingFunctionName = Just "interpretAggregation"- , errorColumnName = Just expression- }- )- t ->- TypeMismatchException- ( MkTypeErrorContext- { userType = Right (typeRep @(VU.Vector b))- , expectedType = Right (typeRep @b)- , callingFunctionName = Just "interpretAggregation"- , errorColumnName = Just expression- }- )- interpretAggregation :: forall a. (Columnable a) =>@@ -461,7 +179,13 @@ Left e -> Left e Right (UnAggregated unaggregated) -> case unaggregated of BoxedColumn (col :: V.Vector b) -> case testEquality (typeRep @b) (typeRep @(V.Vector c)) of- Just Refl -> Right $ UnAggregated $ fromVector $ V.map (V.map f) col+ Just Refl -> case sUnbox @d of+ SFalse -> Right $ UnAggregated $ fromVector $ V.map (V.map f) col+ STrue ->+ Right $+ UnAggregated $+ fromVector $+ V.map (V.convert @V.Vector @d @VU.Vector . V.map f) col Nothing -> case testEquality (typeRep @b) (typeRep @(VU.Vector c)) of Nothing -> Left $ nestedTypeException @b @c (show expression) Just Refl -> case (sUnbox @c, sUnbox @a) of@@ -472,6 +196,30 @@ Right (Aggregated (TColumn aggregated)) -> case mapColumn f aggregated of Left e -> Left e Right col -> Right $ Aggregated $ TColumn col+interpretAggregation gdf expression@(BinaryOp name (f :: c -> d -> e) left (Lit (right :: g))) = case testEquality (typeRep @g) (typeRep @d) of+ Just Refl -> interpretAggregation gdf (UnaryOp name (`f` right) left)+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @g)+ , expectedType = Right (typeRep @d)+ , callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ )+interpretAggregation gdf expression@(BinaryOp name (f :: c -> d -> e) (Lit (left :: g)) right) = case testEquality (typeRep @g) (typeRep @c) of+ Just Refl -> interpretAggregation gdf (UnaryOp name (f left) right)+ Nothing ->+ Left $+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @g)+ , expectedType = Right (typeRep @c)+ , callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ ) interpretAggregation gdf expression@(BinaryOp _ (f :: c -> d -> e) left right) = case (interpretAggregation @c gdf left, interpretAggregation @d gdf right) of (Right (Aggregated (TColumn left')), Right (Aggregated (TColumn right'))) -> case zipWithColumns f left' right' of@@ -517,19 +265,17 @@ V.zipWith (\l' r' -> V.zipWith f l' (V.convert r')) l r (_, _) -> Left $ InternalException "Unboxed vectors contain boxed types" Nothing -> case testEquality (typeRep @n) (typeRep @(V.Vector d)) of- Just Refl -> case sUnbox @c of- STrue -> case sUnbox @e of- SFalse ->- Right $- UnAggregated $- fromVector $- V.zipWith (V.zipWith f . V.convert) l r- STrue ->- Right $- UnAggregated $- fromVector $- V.zipWith (\l' r' -> V.convert @V.Vector @e @VU.Vector $ V.zipWith f l' r') l r- SFalse -> Left $ InternalException "Unboxed vectors contain boxed types"+ Just Refl -> case sUnbox @e of+ SFalse ->+ Right $+ UnAggregated $+ fromVector $+ V.zipWith (V.zipWith f . V.convert) l r+ STrue ->+ Right $+ UnAggregated $+ fromVector $+ V.zipWith (\l' r' -> V.convert @V.Vector @e @VU.Vector $ V.zipWith f l' r') l r Nothing -> Left $ nestedTypeException @n @d (show right) _ -> Left $ InternalException "Aggregated into a non-boxed column" (Right _, Right _) ->@@ -553,6 +299,205 @@ } ) (_, Left e) -> Left e+interpretAggregation gdf expression@(If cond (Lit l) (Lit r)) =+ case interpretAggregation @Bool gdf cond of+ Right (Aggregated (TColumn conditions)) -> case mapColumn+ (\(c :: Bool) -> if c then l else r)+ conditions of+ Left e -> Left e+ Right v -> Right $ Aggregated (TColumn v)+ Right (UnAggregated conditions) -> case sUnbox @a of+ STrue -> case mapColumn+ (\(c :: VU.Vector Bool) -> VU.map (\c' -> if c' then l else r) c)+ conditions of+ Left (TypeMismatchException context) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ )+ Left e -> Left e+ Right v -> Right $ UnAggregated v+ SFalse -> case mapColumn+ (\(c :: VU.Vector Bool) -> V.map (\c' -> if c' then l else r) (VU.convert c))+ conditions of+ Left (TypeMismatchException context) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ )+ Left e -> Left e+ Right v -> Right $ UnAggregated v+ Left (TypeMismatchException context) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show cond)+ }+ )+ Left e -> Left e+interpretAggregation gdf expression@(If cond (Lit l) r) =+ case ( interpretAggregation @Bool gdf cond+ , interpretAggregation @a gdf r+ ) of+ ( Right (Aggregated (TColumn conditions))+ , Right (Aggregated (TColumn right))+ ) -> case zipWithColumns+ (\(c :: Bool) (r' :: a) -> if c then l else r')+ conditions+ right of+ Left e -> Left e+ Right v -> Right $ Aggregated (TColumn v)+ ( Right (UnAggregated conditions)+ , Right (UnAggregated right@(BoxedColumn (right' :: V.Vector c)))+ ) -> case testEquality (typeRep @(V.Vector a)) (typeRep @c) of+ Just Refl -> case zipWithColumns+ ( \(c :: VU.Vector Bool) (r' :: V.Vector a) ->+ V.zipWith+ (\c' r'' -> if c' then l else r'')+ (V.convert c)+ r'+ )+ conditions+ right of+ Left (TypeMismatchException context) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ )+ Left e -> Left e+ Right v -> Right $ UnAggregated v+ Nothing -> case testEquality (typeRep @(VU.Vector a)) (typeRep @c) of+ Nothing -> Left $ nestedTypeException @c @a (show expression)+ Just Refl -> case sUnbox @a of+ SFalse -> Left $ InternalException "Boxed type in unboxed column"+ STrue -> case zipWithColumns+ ( \(c :: VU.Vector Bool) (r' :: VU.Vector a) ->+ VU.zipWith+ (\c' r'' -> if c' then l else r'')+ c+ r'+ )+ conditions+ right of+ Left (TypeMismatchException context) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ )+ Left e -> Left e+ Right v -> Right $ UnAggregated v+ (Right _, Right _) ->+ Left $+ AggregatedAndNonAggregatedException (T.pack $ show l) (T.pack $ show r)+ (Left (TypeMismatchException context), _) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show cond)+ }+ )+ (Left e, _) -> Left e+ (_, Left (TypeMismatchException context)) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show r)+ }+ )+ (_, Left e) -> Left e+interpretAggregation gdf expression@(If cond l (Lit r)) =+ case ( interpretAggregation @Bool gdf cond+ , interpretAggregation @a gdf l+ ) of+ ( Right (Aggregated (TColumn conditions))+ , Right (Aggregated (TColumn left))+ ) -> case zipWithColumns+ (\(c :: Bool) (l' :: a) -> if c then l' else r)+ conditions+ left of+ Left e -> Left e+ Right v -> Right $ Aggregated (TColumn v)+ ( Right (UnAggregated conditions)+ , Right (UnAggregated left@(BoxedColumn (left' :: V.Vector c)))+ ) -> case testEquality (typeRep @(V.Vector a)) (typeRep @c) of+ Just Refl -> case zipWithColumns+ ( \(c :: VU.Vector Bool) (l' :: V.Vector a) ->+ V.zipWith+ (\c' l'' -> if c' then l'' else r)+ (V.convert c)+ l'+ )+ conditions+ left of+ Left (TypeMismatchException context) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ )+ Left e -> Left e+ Right v -> Right $ UnAggregated v+ Nothing -> case testEquality (typeRep @(VU.Vector a)) (typeRep @c) of+ Nothing -> Left $ nestedTypeException @c @a (show expression)+ Just Refl -> case sUnbox @a of+ SFalse -> Left $ InternalException "Boxed type in unboxed column"+ STrue -> case zipWithColumns+ ( \(c :: VU.Vector Bool) (l' :: VU.Vector a) ->+ VU.zipWith+ (\c' l'' -> if c' then l'' else r)+ c+ l'+ )+ conditions+ left of+ Left (TypeMismatchException context) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show expression)+ }+ )+ Left e -> Left e+ Right v -> Right $ UnAggregated v+ (Right _, Right _) ->+ Left $+ AggregatedAndNonAggregatedException (T.pack $ show l) (T.pack $ show r)+ (Left (TypeMismatchException context), _) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show cond)+ }+ )+ (Left e, _) -> Left e+ (_, Left (TypeMismatchException context)) ->+ Left $+ TypeMismatchException+ ( context+ { callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just (show r)+ }+ )+ (_, Left e) -> Left e interpretAggregation gdf expression@(If cond l r) = case ( interpretAggregation @Bool gdf cond , interpretAggregation @a gdf l@@ -845,20 +790,18 @@ Aggregated $ TColumn $ fromVector $- V.map (\v -> VU.foldl' f (VG.head v) (VG.drop 1 v)) col+ V.map (VU.foldl1' f) col SFalse -> Left $ InternalException "Boxed type inside an unboxed column" Just Refl -> Right $ Aggregated $ TColumn $ fromVector $- V.map (\v -> VG.foldl' f (VG.head v) (VG.drop 1 v)) col+ V.map (VG.foldl1' f) col Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column"- Right (Aggregated (TColumn column)) -> case headColumn @a column of+ Right (Aggregated (TColumn column)) -> case foldl1Column f column of Left e -> Left e- Right h -> case ifoldlColumn (\acc _ v -> f acc v) h column of- Left e -> Left e- Right value -> interpretAggregation @a gdf (Lit value)+ Right value -> interpretAggregation @a gdf (Lit value) interpretAggregation gdf@(Grouped df names indices os) expression@(AggFold expr op s (f :: (a -> b -> a))) = case interpretAggregation @b gdf expr of (Left (TypeMismatchException context)) ->@@ -879,17 +822,15 @@ SFalse -> Left $ InternalException "Boxed type inside an unboxed column" Nothing -> Left $ nestedTypeException @d @b (show expr) Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column"- Right (Aggregated (TColumn column)) -> case ifoldlColumn (\acc _ v -> f acc v) s column of+ Right (Aggregated (TColumn column)) -> case foldlColumn f s column of Left e -> Left e Right value -> interpretAggregation @a gdf (Lit value) instance (Num a, Columnable a) => Num (Expr a) where (+) :: Expr a -> Expr a -> Expr a (+) (Lit x) (Lit y) = Lit (x + y)- (+) (Lit x) expr = UnaryOp ("add " <> (T.pack . show) (Lit x)) (+ x) expr- (+) expr (Lit x) = UnaryOp ("add " <> (T.pack . show) (Lit x)) (+ x) expr (+) e1 e2- | e1 == e2 = UnaryOp ("mult " <> (T.pack . show) (Lit @a 2)) (* 2) e1+ | e1 == e2 = BinaryOp "mult" (*) e1 (Lit 2) | otherwise = BinaryOp "add" (+) e1 e2 (-) :: Expr a -> Expr a -> Expr a@@ -902,8 +843,6 @@ (*) (Lit 1) e = e (*) e (Lit 1) = e (*) (Lit x) (Lit y) = Lit (x * y)- (*) (Lit x) expr = UnaryOp ("mult " <> (T.pack . show) (Lit x)) (* x) expr- (*) expr (Lit x) = UnaryOp ("mult " <> (T.pack . show) (Lit x)) (* x) expr (*) e1 e2 | e1 == e2 = UnaryOp "pow 2" (^ 2) e1 | otherwise = BinaryOp "mult" (*) e1 e2@@ -913,18 +852,11 @@ negate :: Expr a -> Expr a negate (Lit n) = Lit (negate n)- negate expr = case expr of- (UnaryOp "negate" negate e) -> case testEquality (typeOf e) (typeOf expr) of- Nothing -> expr -- This case is impossible.- Just Refl -> e- _ -> UnaryOp "negate" negate expr+ negate expr = UnaryOp "negate" negate expr abs :: (Num a) => Expr a -> Expr a- abs expr = case expr of- (UnaryOp "abs" abs e) -> case testEquality (typeOf e) (typeOf expr) of- Nothing -> expr -- This case is impossible.- Just Refl -> e- _ -> UnaryOp "abs" abs expr+ abs (Lit n) = Lit (abs n)+ abs expr = UnaryOp "abs" abs expr signum :: (Num a) => Expr a -> Expr a signum (Lit n) = Lit (signum n)@@ -944,7 +876,8 @@ fromRational = Lit . fromRational (/) :: (Fractional a, Columnable a) => Expr a -> Expr a -> Expr a- (/) = BinaryOp "divide" (/)+ (/) (Lit l1) (Lit l2) = Lit (l1 / l2)+ (/) e1 e2 = BinaryOp "divide" (/) e1 e2 divide :: (Fractional a, Columnable a) => Expr a -> Expr a -> Expr a divide = (/)@@ -1161,3 +1094,139 @@ numRows :: DataFrame -> Int numRows df = fst (dataframeDimensions df)++mkUnaggregatedColumnBoxed ::+ forall a.+ (Columnable a) =>+ V.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (V.Vector a)+mkUnaggregatedColumnBoxed col os indices =+ let+ sorted = V.unsafeBackpermute col (V.convert indices)+ n i = os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)+ start i = os `VG.unsafeIndex` i+ in+ V.generate+ (VU.length os - 1)+ ( \i ->+ V.unsafeSlice (start i) (n i) sorted+ )++mkUnaggregatedColumnUnboxed ::+ forall a.+ (Columnable a, VU.Unbox a) =>+ VU.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (VU.Vector a)+mkUnaggregatedColumnUnboxed col os indices =+ let+ sorted = VU.unsafeBackpermute col indices+ n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i)+ start i = os `VG.unsafeIndex` i+ in+ V.generate+ (VU.length os - 1)+ ( \i ->+ VU.unsafeSlice (start i) (n i) sorted+ )++mkAggregatedColumnUnboxed ::+ forall a b.+ (Columnable a, VU.Unbox a, Columnable b, VU.Unbox b) =>+ VU.Vector a ->+ VU.Vector Int ->+ VU.Vector Int ->+ (VU.Vector a -> b) ->+ VU.Vector b+mkAggregatedColumnUnboxed col os indices f =+ let+ sorted = VU.unsafeBackpermute col indices+ n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i)+ start i = os `VG.unsafeIndex` i+ in+ VU.generate+ (VU.length os - 1)+ ( \i ->+ f (VU.unsafeSlice (start i) (n i) sorted)+ )++mkReducedColumnUnboxed ::+ forall a.+ (VU.Unbox a) =>+ VU.Vector a ->+ VU.Vector Int ->+ VU.Vector Int ->+ (a -> a -> a) ->+ VU.Vector a+mkReducedColumnUnboxed col os indices f = runST $ do+ let len = VU.length os - 1+ mvec <- VUM.unsafeNew len++ let loopOut i+ | i == len = return ()+ | otherwise = do+ let start = os `VU.unsafeIndex` i+ let end = os `VU.unsafeIndex` (i + 1)+ let initVal = col `VU.unsafeIndex` (indices `VU.unsafeIndex` start)++ let loopIn !acc idx+ | idx == end = acc+ | otherwise =+ let val = col `VU.unsafeIndex` (indices `VU.unsafeIndex` idx)+ in loopIn (f acc val) (idx + 1)+ let !finalVal = loopIn initVal (start + 1)+ VUM.unsafeWrite mvec i finalVal+ loopOut (i + 1)++ loopOut 0+ VU.unsafeFreeze mvec+{-# INLINE mkReducedColumnUnboxed #-}++mkReducedColumnBoxed ::+ V.Vector a ->+ VU.Vector Int ->+ VU.Vector Int ->+ (a -> a -> a) ->+ V.Vector a+mkReducedColumnBoxed col os indices f = runST $ do+ let len = VU.length os - 1+ mvec <- VM.unsafeNew len++ let loopOut i+ | i == len = return ()+ | otherwise = do+ let start = os `VU.unsafeIndex` i+ let end = os `VU.unsafeIndex` (i + 1)+ let initVal = col `V.unsafeIndex` (indices `VU.unsafeIndex` start)++ let loopIn !acc idx+ | idx == end = acc+ | otherwise =+ let val = col `V.unsafeIndex` (indices `VU.unsafeIndex` idx)+ in loopIn (f acc val) (idx + 1)+ let !finalVal = loopIn initVal (start + 1)+ VM.unsafeWrite mvec i finalVal+ loopOut (i + 1)++ loopOut 0+ V.unsafeFreeze mvec+{-# INLINE mkReducedColumnBoxed #-}++nestedTypeException ::+ forall a b. (Typeable a, Typeable b) => String -> DataFrameException+nestedTypeException expression = case typeRep @a of+ App t1 t2 ->+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Left (show (typeRep @b)) :: Either String (TypeRep ())+ , expectedType = Left (show (typeRep @a)) :: Either String (TypeRep ())+ , callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just expression+ }+ )+ t ->+ TypeMismatchException+ ( MkTypeErrorContext+ { userType = Right (typeRep @(VU.Vector b))+ , expectedType = Right (typeRep @b)+ , callingFunctionName = Just "interpretAggregation"+ , errorColumnName = Just expression+ }+ )
src/DataFrame/Internal/Row.hs view
@@ -110,7 +110,7 @@ ==== __Examples__ >>> toRowList df-[Row {name = "Alice", age = 25, ...}, Row {name = "Bob", age = 30, ...}, ...]+[[("name", "Alice"), ("age", 25), ...], [("name", "Bob"), ("age", 30), ...], ...] ==== __Performance note__ @@ -118,12 +118,14 @@ for large dataframes. Consider using 'toRowVector' if you need random access or streaming operations. -}-toRowList :: DataFrame -> [Row]+toRowList :: DataFrame -> [[(T.Text, Any)]] toRowList df = let names = map fst (L.sortBy (compare `on` snd) $ M.toList (columnIndices df)) in- map (mkRowRep df names) [0 .. (fst (dataframeDimensions df) - 1)]+ map+ (zip names . V.toList . mkRowRep df names)+ [0 .. (fst (dataframeDimensions df) - 1)] {- | Converts the dataframe to a vector of rows with only the specified columns.
src/DataFrame/Internal/Statistics.hs view
@@ -1,9 +1,12 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-} module DataFrame.Internal.Statistics where +import qualified Data.Vector as V import qualified Data.Vector.Algorithms.Intro as VA+import qualified Data.Vector.Mutable as VM import qualified Data.Vector.Unboxed as VU import qualified Data.Vector.Unboxed.Mutable as VUM @@ -14,9 +17,15 @@ mean' :: (Real a, VU.Unbox a) => VU.Vector a -> Double mean' samp | VU.null samp = throw $ EmptyDataSetException "mean"- | otherwise = VU.sum (VU.map realToFrac samp) / fromIntegral (VU.length samp)+ | otherwise = rtf (VU.sum samp) / fromIntegral (VU.length samp) {-# INLINE mean' #-} +meanDouble' :: VU.Vector Double -> Double+meanDouble' samp+ | VU.null samp = throw $ EmptyDataSetException "mean"+ | otherwise = VU.sum samp / fromIntegral (VU.length samp)+{-# INLINE meanDouble' #-}+ median' :: (Real a, VU.Unbox a) => VU.Vector a -> Double median' samp | VU.null samp = throw $ EmptyDataSetException "median"@@ -27,21 +36,23 @@ middleIndex = len `div` 2 middleElement <- VUM.read mutableSamp middleIndex if odd len- then pure (realToFrac middleElement)+ then pure (rtf middleElement) else do prev <- VUM.read mutableSamp (middleIndex - 1)- pure (realToFrac (middleElement + prev) / 2)+ pure (rtf (middleElement + prev) / 2) {-# INLINE median' #-} -- accumulator: count, mean, m2-data VarAcc = VarAcc !Int !Double !Double deriving (Show)+data VarAcc+ = VarAcc {-# UNPACK #-} !Int {-# UNPACK #-} !Double {-# UNPACK #-} !Double+ deriving (Show) -varianceStep :: (Real a) => VarAcc -> a -> VarAcc+varianceStep :: VarAcc -> Double -> VarAcc varianceStep (VarAcc !n !mean !m2) !x = let !n' = n + 1- !delta = realToFrac x - mean+ !delta = x - mean !mean' = mean + delta / fromIntegral n'- !m2' = m2 + delta * (realToFrac x - mean')+ !m2' = m2 + delta * (x - mean') in VarAcc n' mean' m2' {-# INLINE varianceStep #-} @@ -52,16 +63,20 @@ {-# INLINE computeVariance #-} variance' :: (Real a, VU.Unbox a) => VU.Vector a -> Double-variance' = computeVariance . VU.foldl' varianceStep (VarAcc 0 0 0)+variance' = computeVariance . VU.foldl' varianceStep (VarAcc 0 0 0) . VU.map rtf {-# INLINE variance' #-} +varianceDouble' :: VU.Vector Double -> Double+varianceDouble' = computeVariance . VU.foldl' varianceStep (VarAcc 0 0 0)+{-# INLINE varianceDouble' #-}+ -- accumulator: count, mean, m2, m3 data SkewAcc = SkewAcc !Int !Double !Double !Double deriving (Show) skewnessStep :: (VU.Unbox a, Num a, Real a) => SkewAcc -> a -> SkewAcc skewnessStep (SkewAcc !n !mean !m2 !m3) !x' = let !n' = n + 1- x = realToFrac x'+ x = rtf x' !k = fromIntegral n' !delta = x - mean !mean' = mean + delta / k@@ -80,31 +95,31 @@ skewness' = computeSkewness . VU.foldl' skewnessStep (SkewAcc 0 0 0 0) {-# INLINE skewness' #-} -correlation' ::- (Real a, VU.Unbox a, Real b, VU.Unbox b) =>- VU.Vector a -> VU.Vector b -> Maybe Double+data CorrelationStats+ = CorrelationStats+ {-# UNPACK #-} !Double+ {-# UNPACK #-} !Double+ {-# UNPACK #-} !Double+ {-# UNPACK #-} !Double+ {-# UNPACK #-} !Double++correlation' :: VU.Vector Double -> VU.Vector Double -> Maybe Double correlation' xs ys+ | n < 2 = Nothing | VU.length xs /= VU.length ys = Nothing- | nI < 2 = Nothing | otherwise =- let !nf = fromIntegral nI- (!sumX, !sumY, !sumSquaredX, !sumSquaredY, !sumXY) = go 0 0 0 0 0 0+ let nf = fromIntegral n+ initial = CorrelationStats 0 0 0 0 0+ (CorrelationStats sumX sumY sumXX sumYY sumXY) = VU.ifoldl' step initial xs+ !num = nf * sumXY - sumX * sumY- !den = sqrt ((nf * sumSquaredX - sumX * sumX) * (nf * sumSquaredY - sumY * sumY))- in pure (num / den)+ !den = sqrt ((nf * sumXX - sumX * sumX) * (nf * sumYY - sumY * sumY))+ in Just (num / den) where- !nI = VU.length xs- go !i !sumX !sumY !sumSquaredX !sumSquaredY !sumXY- | i < nI =- let !x = realToFrac (VU.unsafeIndex xs i)- !y = realToFrac (VU.unsafeIndex ys i)- !sumX' = sumX + x- !sumY' = sumY + y- !sumSquaredX' = sumSquaredX + x * x- !sumSquaredY' = sumSquaredY + y * y- !sumXY' = sumXY + x * y- in go (i + 1) sumX' sumY' sumSquaredX' sumSquaredY' sumXY'- | otherwise = (sumX, sumY, sumSquaredX, sumSquaredY, sumXY)+ n = VU.length xs+ step (CorrelationStats sx sy sxx syy sxy) i x =+ let !y = VU.unsafeIndex ys i+ in CorrelationStats (sx + x) (sy + y) (sxx + x * x) (syy + y * y) (sxy + x * y) {-# INLINE correlation' #-} quantiles' ::@@ -124,11 +139,11 @@ !position = p * fromIntegral (n - 1) !index = floor position !f = position - fromIntegral index- x <- fmap realToFrac (VUM.read mutableSamp index)+ x <- fmap rtf (VUM.read mutableSamp index) if f == 0 then return x else do- y <- fmap realToFrac (VUM.read mutableSamp (index + 1))+ y <- fmap rtf (VUM.read mutableSamp (index + 1)) return $ (1 - f) * x + f * y ) qs@@ -137,6 +152,31 @@ percentile' :: (VU.Unbox a, Num a, Real a) => Int -> VU.Vector a -> Double percentile' n = VU.head . quantiles' (VU.fromList [n]) 100 +quantilesOrd' ::+ (Ord a, Eq a) =>+ VU.Vector Int -> Int -> V.Vector a -> V.Vector a+quantilesOrd' qs q samp+ | V.null samp = throw $ EmptyDataSetException "quantiles"+ | q < 2 = throw $ WrongQuantileNumberException q+ | VU.any (\i -> i < 0 || i > q) qs = throw $ WrongQuantileIndexException qs q+ | otherwise = runST $ do+ let !n = V.length samp+ mutableSamp <- V.thaw samp+ VA.sort mutableSamp+ V.mapM+ ( \i -> do+ let !p = fromIntegral i / fromIntegral q+ !position = p * fromIntegral (n - 1)+ !index = floor position+ -- This is not exact for Ord instances.+ -- Figure out how to make it so.+ VM.read mutableSamp index+ )+ (V.convert qs)++percentileOrd' :: (Ord a, Eq a) => Int -> V.Vector a -> a+percentileOrd' n = V.head . quantilesOrd' (VU.fromList [n]) 100+ interQuartileRange' :: (VU.Unbox a, Num a, Real a) => VU.Vector a -> Double interQuartileRange' samp = let quartiles = quantiles' (VU.fromList [1, 3]) 4 samp@@ -217,3 +257,11 @@ where clamp z a b = max a (min b z) {-# INLINE jointBincount #-}++rtf :: (Real a) => a -> Double+rtf = realToFrac+{-# NOINLINE [1] rtf #-}++{-# RULES+"rtf/Double" [2] forall (x :: Double). rtf x = x+ #-}
src/DataFrame/Operations/Core.hs view
@@ -26,6 +26,7 @@ import DataFrame.Internal.Column ( Column (..), Columnable,+ TypedColumn (..), columnLength, columnTypeString, expandColumn,@@ -35,12 +36,13 @@ toFloatVector, toIntVector, toUnboxedVector,+ toVector, ) import DataFrame.Internal.DataFrame ( DataFrame (..),+ columnIndices, empty, getColumn,- unsafeGetColumn, ) import DataFrame.Internal.Expression import DataFrame.Internal.Parsing (isNullish)@@ -676,84 +678,75 @@ @ -} valueCounts :: forall a. (Columnable a) => Expr a -> DataFrame -> [(a, Int)]-valueCounts (Col columnName) df = case getColumn columnName df of- Nothing ->- throw $- ColumnNotFoundException columnName "valueCounts" (M.keys $ columnIndices df)- Just (BoxedColumn (column' :: V.Vector c)) ->- let- column = V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column'- in- case (typeRep @a) `testEquality` (typeRep @c) of- Nothing ->- throw $- TypeMismatchException- ( MkTypeErrorContext- { userType = Right $ typeRep @a- , expectedType = Right $ typeRep @c- , errorColumnName = Just (T.unpack columnName)- , callingFunctionName = Just "valueCounts"- }- )- Just Refl -> M.toAscList column- Just (OptionalColumn (column' :: V.Vector c)) ->+valueCounts expr df = case columnAsVector expr df of+ Left e -> throw e+ Right column' -> let column = V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column' in- case (typeRep @a) `testEquality` (typeRep @c) of- Nothing ->- throw $- TypeMismatchException- ( MkTypeErrorContext- { userType = Right $ typeRep @a- , expectedType = Right $ typeRep @c- , errorColumnName = Just (T.unpack columnName)- , callingFunctionName = Just "valueCounts"- }- )- Just Refl -> M.toAscList column- Just (UnboxedColumn (column' :: VU.Vector c)) ->+ M.toAscList column++{- | O (k * n) Shows the proportions of each value in a given column.++==== __Example__+@+>>> df = D.fromUnnamedColumns [D.fromList [1..10], D.fromList [11..20]]++>>> D.valueCounts @Int "0" df++[(1,0.1),(2,0.1),(3,0.1),(4,0.1),(5,0.1),(6,0.1),(7,0.1),(8,0.1),(9,0.1),(10,0.1)]++@+-}+valueProportions ::+ forall a. (Columnable a) => Expr a -> DataFrame -> [(a, Double)]+valueProportions expr df = case columnAsVector expr df of+ Left e -> throw e+ Right column' -> let- column =- V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty (V.convert column')+ counts =+ M.toAscList+ (V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column')+ total = fromIntegral (sum (map snd counts)) in- case (typeRep @a) `testEquality` (typeRep @c) of- Nothing ->- throw $- TypeMismatchException- ( MkTypeErrorContext- { userType = Right $ typeRep @a- , expectedType = Right $ typeRep @c- , errorColumnName = Just (T.unpack columnName)- , callingFunctionName = Just "valueCounts"- }- )- Just Refl -> M.toAscList column-valueCounts _ _ = error "Cannot call value counts on non-column reference"+ map (fmap ((/ total) . fromIntegral)) counts {- | A left fold for dataframes that takes the dataframe as the last object. This makes it easier to chain operations. ==== __Example__ @->>> D.fold (const id) [1..5] df+>>> df = D.fromNamedColumns [("x", D.fromList [1..100]), ("y", D.fromList [11..110])]+>>> D.fold D.dropLast [1..5] df ------------ 0 | 1------|----- Int | Int------|----- 1 | 11- 2 | 12- 3 | 13- 4 | 14- 5 | 15- 6 | 16- 7 | 17- 8 | 18- 9 | 19- 10 | 20+---------+ x | y+----|----+Int | Int+----|----+1 | 11+2 | 12+3 | 13+4 | 14+5 | 15+6 | 16+7 | 17+8 | 18+9 | 19+10 | 20+11 | 21+12 | 22+13 | 23+14 | 24+15 | 25+16 | 26+17 | 27+18 | 28+19 | 29+20 | 30 +Showing 20 rows out of 85+ @ -} fold :: (a -> DataFrame -> DataFrame) -> [a] -> DataFrame -> DataFrame@@ -845,28 +838,22 @@ ==== __Examples__ ->>> columnAsVector @Int "age" df-[25, 30, 35, ...]-->>> columnAsVector @Text "name" df-["Alice", "Bob", "Charlie", ...]--==== __Throws__+>>> columnAsVector (F.col @Int "age") df+Right [25, 30, 35, ...] -* 'error' - if the column type doesn't match the requested type+>>> columnAsVector (F.col @Text "name") df+Right ["Alice", "Bob", "Charlie", ...] -}-columnAsVector :: forall a. (Columnable a) => Expr a -> DataFrame -> V.Vector a-columnAsVector (Col name) df = case unsafeGetColumn name df of- (BoxedColumn (col :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> error "Type error"- Just Refl -> col- (OptionalColumn (col :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> error "Type error"- Just Refl -> col- (UnboxedColumn (col :: VU.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of- Nothing -> error "Type error"- Just Refl -> VG.convert col-columnAsVector _ _ = error "UNIMPLEMENTED"+columnAsVector ::+ forall a.+ (Columnable a) => Expr a -> DataFrame -> Either DataFrameException (V.Vector a)+columnAsVector (Col name) df = case getColumn name df of+ Just col -> toVector col+ Nothing ->+ Left $ ColumnNotFoundException name "columnAsVector" (M.keys $ columnIndices df)+columnAsVector expr df = case interpret df expr of+ Left e -> throw e+ Right (TColumn col) -> toVector col {- | Retrieves a column as an unboxed vector of 'Int' values. @@ -875,8 +862,14 @@ -} columnAsIntVector :: Expr Int -> DataFrame -> Either DataFrameException (VU.Vector Int)-columnAsIntVector (Col name) df = toIntVector (unsafeGetColumn name df)-columnAsIntVector _ _ = error "UNIMPLEMENTED"+columnAsIntVector (Col name) df = case getColumn name df of+ Just col -> toIntVector col+ Nothing ->+ Left $+ ColumnNotFoundException name "columnAsIntVector" (M.keys $ columnIndices df)+columnAsIntVector expr df = case interpret df expr of+ Left e -> throw e+ Right (TColumn col) -> toIntVector col {- | Retrieves a column as an unboxed vector of 'Double' values. @@ -885,8 +878,14 @@ -} columnAsDoubleVector :: Expr Double -> DataFrame -> Either DataFrameException (VU.Vector Double)-columnAsDoubleVector (Col name) df = toDoubleVector (unsafeGetColumn name df)-columnAsDoubleVector _ _ = error "UNIMPLEMENTED"+columnAsDoubleVector (Col name) df = case getColumn name df of+ Just col -> toDoubleVector col+ Nothing ->+ Left $+ ColumnNotFoundException name "columnAsDoubleVector" (M.keys $ columnIndices df)+columnAsDoubleVector expr df = case interpret df expr of+ Left e -> throw e+ Right (TColumn col) -> toDoubleVector col {- | Retrieves a column as an unboxed vector of 'Float' values. @@ -895,15 +894,31 @@ -} columnAsFloatVector :: Expr Float -> DataFrame -> Either DataFrameException (VU.Vector Float)-columnAsFloatVector (Col name) df = toFloatVector (unsafeGetColumn name df)-columnAsFloatVector _ _ = error "UNIMPLEMENTED"+columnAsFloatVector (Col name) df = case getColumn name df of+ Just col -> toFloatVector col+ Nothing ->+ Left $+ ColumnNotFoundException name "columnAsFloatVector" (M.keys $ columnIndices df)+columnAsFloatVector expr df = case interpret df expr of+ Left e -> throw e+ Right (TColumn col) -> toFloatVector col columnAsUnboxedVector :: forall a. (Columnable a, VU.Unbox a) => Expr a -> DataFrame -> Either DataFrameException (VU.Vector a)-columnAsUnboxedVector (Col name) df = toUnboxedVector @a (unsafeGetColumn name df)-columnAsUnboxedVector _ _ = error "UNIMPLEMENTED"+columnAsUnboxedVector (Col name) df = case getColumn name df of+ Just col -> toUnboxedVector col+ Nothing ->+ Left $+ ColumnNotFoundException name "columnAsFloatVector" (M.keys $ columnIndices df)+columnAsUnboxedVector expr df = case interpret df expr of+ Left e -> throw e+ Right (TColumn col) -> toUnboxedVector col+{-# SPECIALIZE columnAsUnboxedVector ::+ Expr Double -> DataFrame -> Either DataFrameException (VU.Vector Double)+ #-}+{-# INLINE columnAsUnboxedVector #-} {- | Get a specific column as a list. @@ -922,5 +937,4 @@ * 'error' - if the column type doesn't match the requested type -} columnAsList :: forall a. (Columnable a) => Expr a -> DataFrame -> [a]-columnAsList expr@(Col name) df = V.toList (columnAsVector expr df)-columnAsList _ _ = error "UNIMPLEMENTED"+columnAsList expr df = either throw V.toList (columnAsVector expr df)
src/DataFrame/Operations/Statistics.hs view
@@ -87,9 +87,9 @@ -- | Calculates the mean of a given column as a standalone value. mean :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-mean (Col name) df = case columnAsUnboxedVector (Col @a name) df of- Right xs -> mean' xs- Left e -> throw e+mean (Col name) df = case _getColumnAsDouble name df of+ Just xs -> meanDouble' xs+ Nothing -> error "[INTERNAL ERROR] Column is non-numeric" mean expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of@@ -98,7 +98,9 @@ meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double-meanMaybe (Col name) df = (mean' . optionalToDoubleVector) (columnAsVector (Col @(Maybe a) name) df)+meanMaybe (Col name) df =+ (mean' . optionalToDoubleVector)+ (either throw id (columnAsVector (Col @(Maybe a) name) df)) meanMaybe expr df = case interpret @(Maybe a) df expr of Left e -> throw e Right (TColumn col) -> case toVector @(Maybe a) col of@@ -144,9 +146,9 @@ -- | Calculates the variance of a given column as a standalone value. variance :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double-variance (Col name) df = case columnAsUnboxedVector (Col @a name) df of- Right xs -> variance' xs- Left e -> throw e+variance (Col name) df = case _getColumnAsDouble name df of+ Just xs -> varianceDouble' xs+ Nothing -> error "[INTERNAL ERROR] Column is non-numeric" variance expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> case toUnboxedVector @a col of@@ -183,8 +185,8 @@ SFalse -> Nothing Nothing -> throw $- ColumnNotFoundException name "applyStatistic" (M.keys $ columnIndices df)- _ -> Nothing+ ColumnNotFoundException name "_getColumnAsDouble" (M.keys $ columnIndices df)+ _ -> Nothing -- Return a type mismatch error here. {-# INLINE _getColumnAsDouble #-} optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double
src/DataFrame/Synthesis.hs view
@@ -16,7 +16,6 @@ import DataFrame.Internal.Column import DataFrame.Internal.DataFrame ( DataFrame (..),- columnAsDoubleVector, ) import DataFrame.Internal.Expression ( Expr (..),@@ -24,6 +23,7 @@ interpret, ) import DataFrame.Internal.Statistics+import DataFrame.Operations.Core (columnAsDoubleVector) import qualified DataFrame.Operations.Statistics as Stats import DataFrame.Operations.Subset (exclude) @@ -48,7 +48,7 @@ let newConds = [ p .<= q- | p <- ps+ | p <- filter (not . isLiteral) ps , q <- ps , p /= q ]@@ -265,7 +265,8 @@ percentiles :: DataFrame -> [Expr Double] percentiles df = let- doubleColumns = map (either throw id . (`columnAsDoubleVector` df)) (D.columnNames df)+ doubleColumns =+ map (either throw id . ((`columnAsDoubleVector` df) . Col)) (D.columnNames df) in concatMap (\c -> map (Lit . roundTo2SigDigits . (`percentile'` c)) [1, 25, 75, 99])@@ -301,7 +302,13 @@ Left e -> throw e Right v -> v cfg = BeamConfig d b MeanSquaredError True- constants = percentiles df' ++ [Lit 10, Lit 1, Lit 0.1, Lit targetMean]+ constants =+ percentiles df'+ ++ [Lit targetMean]+ ++ [ F.pow p i+ | i <- [1 .. 6]+ , p <- [Lit 10, Lit 1, Lit 0.1]+ ] in case beamSearch df' cfg t constants [] [] of Nothing -> Left "No programs found"@@ -364,7 +371,7 @@ (generatePrograms (includeConditionals cfg) conditions vars constants ps) where vars = map Col names- conditions = generateConditions outputs conds vars df+ conditions = generateConditions outputs conds (vars ++ constants) df ps = pickTopN df outputs cfg $ deduplicate df programs names = (map fst . L.sortBy (compare `on` snd) . M.toList . columnIndices) df