diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Revision history for dataframe +## 0.3.3.0 +* Better error messaging for expression failures. +* Fix bug where exponentials were not being properly during CSV parsing. +* `toMatrix` now returns either an exception or the a vector of vector doubles. +* Add `sample`, `kFolds`, and `randomSplit` to sample dataframes. + ## 0.3.2.0 * Fix dataframe semigroup instance. Appending two rows of the same name but different types now gives a row of `Either a b` (work by @jhrcek). * Fix left expansion of semigroup instance (work by @jhrcek). diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 0.3.2.0 +version: 0.3.3.0 synopsis: A fast, safe, and intuitive DataFrame library. @@ -117,7 +117,6 @@ dataframe ^>= 0.3 default-language: Haskell2010 ghc-options: - -O2 -threaded -rtsopts -with-rtsopts=-N diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -278,7 +278,10 @@ filterJust, filterNothing, filterWhere, + kFolds, + randomSplit, range, + sample, select, selectBy, take, diff --git a/src/DataFrame/Errors.hs b/src/DataFrame/Errors.hs --- a/src/DataFrame/Errors.hs +++ b/src/DataFrame/Errors.hs @@ -28,8 +28,12 @@ (Typeable a, Typeable b) => TypeErrorContext a b -> DataFrameException + AggregatedAndNonAggregatedException :: T.Text -> T.Text -> DataFrameException ColumnNotFoundException :: T.Text -> T.Text -> [T.Text] -> DataFrameException EmptyDataSetException :: T.Text -> DataFrameException + InternalException :: T.Text -> DataFrameException + NonColumnReferenceException :: T.Text -> DataFrameException + UnaggregatedException :: T.Text -> DataFrameException WrongQuantileNumberException :: Int -> DataFrameException WrongQuantileIndexException :: VU.Vector Int -> Int -> DataFrameException deriving (Exception) @@ -51,6 +55,14 @@ show (EmptyDataSetException callPoint) = emptyDataSetError callPoint show (WrongQuantileNumberException q) = wrongQuantileNumberError q show (WrongQuantileIndexException qs q) = wrongQuantileIndexError qs q + show (InternalException msg) = "Internal error: " ++ T.unpack msg + show (NonColumnReferenceException msg) = "Expression must be a column reference in: " ++ T.unpack msg + show (UnaggregatedException expr) = "Expression is not fully aggregated: " ++ T.unpack expr + show (AggregatedAndNonAggregatedException expr1 expr2) = + "Cannot combine aggregated and non-aggregated expressions: \n" + ++ T.unpack expr1 + ++ "\n" + ++ T.unpack expr2 columnNotFound :: T.Text -> T.Text -> [T.Text] -> String columnNotFound name callPoint columns = @@ -102,31 +114,21 @@ err ++ ( "\n\tThis happened when calling function " ++ brightGreen cp - ++ " on the column " + ++ " on " ++ brightGreen name - ++ "\n\n" - ++ typeAnnotationSuggestion cp ) -addCallPointInfo Nothing (Just cp) err = err ++ "\n" ++ typeAnnotationSuggestion cp +addCallPointInfo Nothing (Just cp) err = + err + ++ ( "\n\tThis happened when calling function " + ++ brightGreen cp + ) addCallPointInfo (Just name) Nothing err = err - ++ ( "\n\tOn the column " + ++ ( "\n\tOn " ++ name ++ "\n\n" ) addCallPointInfo Nothing Nothing err = err - -typeAnnotationSuggestion :: String -> String -typeAnnotationSuggestion cp = - "\n\n\tTry adding a type at the end of the function e.g " - ++ "change\n\t\t" - ++ red (cp ++ " ...") - ++ " to \n\t\t" - ++ green ("(" ++ cp ++ " ... :: )") - ++ "\n\tor add " - ++ "{-# LANGUAGE TypeApplications #-} to the top of your " - ++ "file then change the call to \n\t\t" - ++ brightGreen (cp ++ " @ ....") guessColumnName :: T.Text -> [T.Text] -> T.Text guessColumnName userInput columns = case map (\k -> (editDistance userInput k, k)) columns of diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs --- a/src/DataFrame/Functions.hs +++ b/src/DataFrame/Functions.hs @@ -27,6 +27,7 @@ import qualified DataFrame.Operations.Statistics as Stats import DataFrame.Operations.Subset (exclude, select) +import Control.Exception (throw) import Control.Monad import qualified Data.Char as Char import Data.Function @@ -293,7 +294,9 @@ | S.member res seen = go seen xs | otherwise = (x, res) : go (S.insert res seen) xs where - res = interpret df x + res = case interpret df x of + Left e -> throw e + Right v -> v hasInvalid = case res of (TColumn (UnboxedColumn (col :: VU.Vector a))) -> case testEquality (typeRep @Double) (typeRep @a) of Just Refl -> VU.any (\n -> isNaN n || isInfinite n) col @@ -302,7 +305,9 @@ -- | Checks if two programs generate the same outputs given all the same inputs. equivalent :: DataFrame -> Expr Double -> Expr Double -> Bool -equivalent df p1 p2 = interpret df p1 Prelude.== interpret df p2 +equivalent df p1 p2 = case (Prelude.==) <$> interpret df p1 <*> interpret df p2 of + Left e -> throw e + Right v -> v synthesizeFeatureExpr :: -- | Target expression @@ -316,11 +321,14 @@ synthesizeFeatureExpr target d b df = let df' = exclude [target] df + t = case interpret df (Col target) of + Left e -> throw e + Right v -> v in case beamSearch df' (BeamConfig d b (\l r -> (^ 2) <$> correlation' l r)) - (interpret df (Col target)) + t [] of Nothing -> Left "No programs found" Just p -> Right p @@ -338,6 +346,9 @@ let df' = exclude [target] df targetMean = fromMaybe 0 $ Stats.mean target df + t = case interpret df (Col target) of + Left e -> throw e + Right v -> v in case beamSearch df' @@ -351,7 +362,7 @@ r ) ) - (interpret df (Col target)) + t [] of Nothing -> Left "No programs found" Just p -> @@ -362,7 +373,7 @@ & select ["_generated_regression_feature_"] ) (BeamConfig d b (\l r -> fmap negate (meanSquaredError l r))) - (interpret df (Col target)) + t [Col "_generated_regression_feature_", lit targetMean, lit 10] of Nothing -> Left "Could not find coefficients" Just p' -> Right (replaceExpr p (Col @Double "_generated_regression_feature_") p') @@ -405,7 +416,9 @@ pickTopN _ _ _ [] = [] pickTopN df (TColumn col) cfg ps = let - l = VU.convert (toVector @Double col) + l = case toVector @Double @VU.Vector col of + Left e -> throw e + Right v -> v ordered = Prelude.take (beamLength cfg) @@ -425,12 +438,18 @@ let (TColumn col') = c in - VU.convert (toVector @Double col') + case toVector @Double @VU.Vector col' of + Left e -> throw e + Right v -> VU.convert v interpretDoubleVector e = let - (TColumn col') = interpret df e + (TColumn col') = case interpret df e of + Left e -> throw e + Right v -> v in - VU.convert (toVector @Double col') + case toVector @Double @VU.Vector col' of + Left e -> throw e + Right v -> VU.convert v in trace ( "Best loss: " @@ -444,7 +463,9 @@ satisfiesExamples :: DataFrame -> TypedColumn Double -> Expr Double -> Bool satisfiesExamples df col expr = let - result = interpret df expr + result = case interpret df expr of + Left e -> throw e + Right v -> v in result Prelude.== col diff --git a/src/DataFrame/IO/Parquet.hs b/src/DataFrame/IO/Parquet.hs --- a/src/DataFrame/IO/Parquet.hs +++ b/src/DataFrame/IO/Parquet.hs @@ -8,11 +8,11 @@ import Data.Bits import qualified Data.ByteString as BSO import Data.Char +import Data.Either import Data.IORef import Data.Int import qualified Data.List as L import qualified Data.Map as M -import Data.Maybe import qualified Data.Text as T import Data.Word import qualified DataFrame.Internal.Column as DI @@ -272,4 +272,4 @@ [] -> pure $ DI.fromList ([] :: [Maybe Int]) (c : cs) -> pure $ - L.foldl' (\l r -> fromMaybe (error "concat failed") (DI.concatColumns l r)) c cs + L.foldl' (\l r -> fromRight (error "concat failed") (DI.concatColumns l r)) c cs diff --git a/src/DataFrame/Internal/Column.hs b/src/DataFrame/Internal/Column.hs --- a/src/DataFrame/Internal/Column.hs +++ b/src/DataFrame/Internal/Column.hs @@ -243,22 +243,49 @@ ) => (b -> c) -> Column -> - Maybe Column + Either DataFrameException Column mapColumn f = \case BoxedColumn (col :: VB.Vector a) | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Just (fromVector @c (VB.map f col)) - | otherwise -> Nothing + Right (fromVector @c (VB.map f col)) + | otherwise -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "mapColumn" + , errorColumnName = Nothing + } + ) OptionalColumn (col :: VB.Vector a) | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Just (fromVector @c (VB.map f col)) - | otherwise -> Nothing + Right (fromVector @c (VB.map f col)) + | otherwise -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "mapColumn" + , errorColumnName = Nothing + } + ) UnboxedColumn (col :: VU.Vector a) | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Just $ case sUnbox @c of + Right $ case sUnbox @c of STrue -> UnboxedColumn (VU.map f col) SFalse -> fromVector @c (VB.map f (VB.convert col)) - | otherwise -> Nothing + | otherwise -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "mapColumn" + , errorColumnName = Nothing + } + ) -- | O(1) Gets the number of elements in the column. columnLength :: Column -> Int @@ -322,16 +349,43 @@ (Columnable a) => (a -> Bool) -> Column -> - Maybe (VU.Vector Int) -findIndices pred (BoxedColumn (column :: VB.Vector b)) = do - Refl <- testEquality (typeRep @a) (typeRep @b) - pure $ VG.convert (VG.findIndices pred column) -findIndices pred (UnboxedColumn (column :: VU.Vector b)) = do - Refl <- testEquality (typeRep @a) (typeRep @b) - pure $ VG.findIndices pred column -findIndices pred (OptionalColumn (column :: VB.Vector (Maybe b))) = do - Refl <- testEquality (typeRep @a) (typeRep @(Maybe b)) - pure $ VG.convert (VG.findIndices pred column) + Either DataFrameException (VU.Vector Int) +findIndices pred (BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> pure $ VG.convert (VG.findIndices pred column) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "findIndices" + , errorColumnName = Nothing + } + ) +findIndices pred (UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> pure $ VG.findIndices pred column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "findIndices" + , errorColumnName = Nothing + } + ) +findIndices pred (OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> pure $ VG.convert (VG.findIndices pred column) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "findIndices" + , errorColumnName = Nothing + } + ) -- | An internal function that returns a vector of how indexes change after a column is sorted. sortedIndexes :: Bool -> Column -> VU.Vector Int @@ -362,89 +416,271 @@ imapColumn :: forall b c. (Columnable b, Columnable c) => - (Int -> b -> c) -> Column -> Maybe Column + (Int -> b -> c) -> Column -> Either DataFrameException Column imapColumn f = \case BoxedColumn (col :: VB.Vector a) | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Just (fromVector @c (VB.imap f col)) - | otherwise -> Nothing + pure (fromVector @c (VB.imap f col)) + | otherwise -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "imapColumn" + , errorColumnName = Nothing + } + ) UnboxedColumn (col :: VU.Vector a) | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Just $ + pure $ case sUnbox @c of STrue -> UnboxedColumn (VU.imap f col) SFalse -> fromVector @c (VB.imap f (VB.convert col)) - | otherwise -> Nothing + | otherwise -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "imapColumn" + , errorColumnName = Nothing + } + ) OptionalColumn (col :: VB.Vector a) | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Just (fromVector @c (VB.imap f col)) - | otherwise -> Nothing + pure (fromVector @c (VB.imap f col)) + | otherwise -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "imapColumn" + , errorColumnName = Nothing + } + ) -- | Filter column with index. ifilterColumn :: - forall a. (Columnable a) => (Int -> a -> Bool) -> Column -> Maybe Column -ifilterColumn f c@(BoxedColumn (column :: VB.Vector b)) = do - Refl <- testEquality (typeRep @a) (typeRep @b) - return $ BoxedColumn $ VG.ifilter f column -ifilterColumn f c@(UnboxedColumn (column :: VU.Vector b)) = do - Refl <- testEquality (typeRep @a) (typeRep @b) - return $ UnboxedColumn $ VG.ifilter f column -ifilterColumn _ _ = Nothing + forall a. + (Columnable a) => + (Int -> a -> Bool) -> Column -> Either DataFrameException Column +ifilterColumn f c@(BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> pure $ BoxedColumn $ VG.ifilter f column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "ifilterColumn" + , errorColumnName = Nothing + } + ) +ifilterColumn f c@(UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> pure $ UnboxedColumn $ VG.ifilter f column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "ifilterColumn" + , errorColumnName = Nothing + } + ) +ifilterColumn f c@(OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> pure $ OptionalColumn $ VG.ifilter f column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "ifilterColumn" + , errorColumnName = Nothing + } + ) -- | Fold (right) column with index. ifoldrColumn :: forall a b. - (Columnable a, Columnable b) => (Int -> a -> b -> b) -> b -> Column -> Maybe b -ifoldrColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = do - Refl <- testEquality (typeRep @a) (typeRep @d) - return $ VG.ifoldr f acc column -ifoldrColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = do - Refl <- testEquality (typeRep @a) (typeRep @d) - return $ VG.ifoldr f acc column -ifoldrColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = do - Refl <- testEquality (typeRep @a) (typeRep @d) - return $ VG.ifoldr f acc column + (Columnable a, Columnable b) => + (Int -> a -> b -> b) -> b -> Column -> Either DataFrameException b +ifoldrColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of + Just Refl -> pure $ VG.ifoldr f acc column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "ifoldrColumn" + , errorColumnName = Nothing + } + ) +ifoldrColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of + Just Refl -> pure $ VG.ifoldr f acc column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "ifoldrColumn" + , errorColumnName = Nothing + } + ) +ifoldrColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of + Just Refl -> pure $ VG.ifoldr f acc column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "ifoldrColumn" + , errorColumnName = Nothing + } + ) -- | Fold (left) column with index. ifoldlColumn :: forall a b. - (Columnable a, Columnable b) => (b -> Int -> a -> b) -> b -> Column -> Maybe b -ifoldlColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = do - Refl <- testEquality (typeRep @a) (typeRep @d) - return $ VG.ifoldl' f acc column -ifoldlColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = do - Refl <- testEquality (typeRep @a) (typeRep @d) - return $ VG.ifoldl' f acc column -ifoldlColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = do - Refl <- testEquality (typeRep @a) (typeRep @d) - return $ VG.ifoldl' f acc column + (Columnable a, Columnable b) => + (b -> Int -> a -> b) -> b -> Column -> Either DataFrameException b +ifoldlColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of + Just Refl -> pure $ VG.ifoldl' f acc column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "ifoldlColumn" + , errorColumnName = Nothing + } + ) +ifoldlColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of + Just Refl -> pure $ VG.ifoldl' f acc column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "ifoldlColumn" + , errorColumnName = Nothing + } + ) +ifoldlColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of + Just Refl -> pure $ VG.ifoldl' f acc column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "ifoldlColumn" + , errorColumnName = Nothing + } + ) -headColumn :: forall a. (Columnable a) => Column -> Maybe a -headColumn (BoxedColumn (col :: VB.Vector b)) = do - Refl <- testEquality (typeRep @a) (typeRep @b) - pure (VG.head col) -headColumn (UnboxedColumn (col :: VU.Vector b)) = do - Refl <- testEquality (typeRep @a) (typeRep @b) - pure (VG.head col) -headColumn (OptionalColumn (col :: VB.Vector b)) = do - Refl <- testEquality (typeRep @a) (typeRep @b) - pure (VG.head col) +headColumn :: forall a. (Columnable a) => Column -> Either DataFrameException a +headColumn (BoxedColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> + if VG.null col + then Left (EmptyDataSetException "headColumn") + else pure (VG.head col) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "headColumn" + , errorColumnName = Nothing + } + ) +headColumn (UnboxedColumn (col :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> + if VG.null col + then Left (EmptyDataSetException "headColumn") + else pure (VG.head col) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "headColumn" + , errorColumnName = Nothing + } + ) +headColumn (OptionalColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> + if VG.null col + then Left (EmptyDataSetException "headColumn") + else pure (VG.head col) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "headColumn" + , errorColumnName = Nothing + } + ) -- | Generic reduce function for all Column types. -reduceColumn :: forall a b. (Columnable a) => (a -> b) -> Column -> Maybe b +reduceColumn :: + forall a b. + (Columnable a, Columnable b) => + (a -> b) -> Column -> Either DataFrameException b {-# SPECIALIZE reduceColumn :: - (VU.Vector (Double, Double) -> Double) -> Column -> Maybe Double - , (VU.Vector Double -> Double) -> Column -> Maybe Double + (VU.Vector (Double, Double) -> Double) -> + Column -> + Either DataFrameException Double + , (VU.Vector Double -> Double) -> Column -> Either DataFrameException Double #-} -reduceColumn f (BoxedColumn (column :: c)) = do - Refl <- testEquality (typeRep @c) (typeRep @a) - pure $ f column -reduceColumn f (UnboxedColumn (column :: c)) = do - Refl <- testEquality (typeRep @c) (typeRep @a) - pure $ f column -reduceColumn f (OptionalColumn (column :: c)) = do - Refl <- testEquality (typeRep @c) (typeRep @a) - pure $ f column +reduceColumn f (BoxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of + Just Refl -> pure $ f column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "reduceColumn" + , errorColumnName = Nothing + } + ) +reduceColumn f (UnboxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of + Just Refl -> pure $ f column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "reduceColumn" + , errorColumnName = Nothing + } + ) +reduceColumn f (OptionalColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of + Just Refl -> pure $ f column + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "reduceColumn" + , errorColumnName = Nothing + } + ) {-# INLINE reduceColumn #-} -- | An internal, column version of zip. @@ -474,20 +710,43 @@ zipWithColumns :: forall a b c. (Columnable a, Columnable b, Columnable c) => - (a -> b -> c) -> Column -> Column -> Maybe Column + (a -> b -> c) -> Column -> Column -> Either DataFrameException Column zipWithColumns f (UnboxedColumn (column :: VU.Vector d)) (UnboxedColumn (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of Just Refl -> case testEquality (typeRep @b) (typeRep @e) of Just Refl -> pure $ case sUnbox @c of STrue -> fromUnboxedVector (VU.zipWith f column other) SFalse -> fromVector $ VB.zipWith f (VG.convert column) (VG.convert other) - Nothing -> Nothing - Nothing -> Nothing -zipWithColumns f left right = - let - left' = toVector @a left - right' = toVector @b right - in - pure $ fromVector $ VB.zipWith f left' right' + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @e) + , callingFunctionName = Just "zipWithColumns" + , errorColumnName = Nothing + } + ) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "zipWithColumns" + , errorColumnName = Nothing + } + ) +zipWithColumns f left right = case toVector @a left of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"}) + Left e -> Left e + Right left' -> case toVector @b right of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"}) + Left e -> Left e + Right right' -> pure $ fromVector $ VB.zipWith f left' right' {-# INLINE zipWithColumns #-} -- Functions for mutable columns (intended for IO). @@ -583,17 +842,53 @@ {- | Concatenates two columns. Returns Nothing if the columns are of different types. -} -concatColumns :: Column -> Column -> Maybe Column +concatColumns :: Column -> Column -> Either DataFrameException Column concatColumns (OptionalColumn left) (OptionalColumn right) = case testEquality (typeOf left) (typeOf right) of - Nothing -> Nothing - Just Refl -> Just (OptionalColumn $ left <> right) + Just Refl -> pure (OptionalColumn $ left <> right) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeOf right) + , expectedType = Right (typeOf left) + , callingFunctionName = Just "concatColumns" + , errorColumnName = Nothing + } + ) concatColumns (BoxedColumn left) (BoxedColumn right) = case testEquality (typeOf left) (typeOf right) of - Nothing -> Nothing - Just Refl -> Just (BoxedColumn $ left <> right) + Just Refl -> pure (BoxedColumn $ left <> right) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeOf right) + , expectedType = Right (typeOf left) + , callingFunctionName = Just "concatColumns" + , errorColumnName = Nothing + } + ) concatColumns (UnboxedColumn left) (UnboxedColumn right) = case testEquality (typeOf left) (typeOf right) of - Nothing -> Nothing - Just Refl -> Just (UnboxedColumn $ left <> right) -concatColumns _ _ = Nothing + Just Refl -> pure (UnboxedColumn $ left <> right) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeOf right) + , expectedType = Right (typeOf left) + , callingFunctionName = Just "concatColumns" + , errorColumnName = Nothing + } + ) +concatColumns left right = + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeOf right) + , expectedType = Right (typeOf left) + , callingFunctionName = Just "concatColumns" + , errorColumnName = Nothing + } + ) {- | Concatenates two columns. @@ -636,23 +931,6 @@ OptionalColumn $ fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right -{- | O(n) Converts a column to a boxed vector. Throws an exception if the wrong type is specified. - -__Examples:__ - -@ -> column = fromList [(1 :: Int), 2, 3, 4] -> toVector @Int column -[1,2,3,4] -> toVector @Double column -exception: ... -@ --} -toVector :: forall a. (Columnable a) => Column -> VB.Vector a -toVector xs = case toVectorSafe xs of - Left err -> throw err - Right val -> val - {- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified. __Examples:__ @@ -666,15 +944,15 @@ @ -} toList :: forall a. (Columnable a) => Column -> [a] -toList xs = case toVectorSafe @a xs of +toList xs = case toVector @a xs of Left err -> throw err Right val -> VB.toList val -- | A safe version of toVector that returns an Either type. -toVectorSafe :: +toVector :: forall a v. (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a) -toVectorSafe column@(OptionalColumn (col :: VB.Vector b)) = +toVector column@(OptionalColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of Just Refl -> Right $ VG.convert col Nothing -> @@ -683,11 +961,11 @@ ( MkTypeErrorContext { userType = Right (typeRep @a) , expectedType = Right (typeRep @b) - , callingFunctionName = Just "toVectorSafe" + , callingFunctionName = Just "toVector" , errorColumnName = Nothing } ) -toVectorSafe (BoxedColumn (col :: VB.Vector b)) = +toVector (BoxedColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of Just Refl -> Right $ VG.convert col Nothing -> @@ -696,11 +974,11 @@ ( MkTypeErrorContext { userType = Right (typeRep @a) , expectedType = Right (typeRep @b) - , callingFunctionName = Just "toVectorSafe" + , callingFunctionName = Just "toVector" , errorColumnName = Nothing } ) -toVectorSafe (UnboxedColumn (col :: VU.Vector b)) = +toVector (UnboxedColumn (col :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of Just Refl -> Right $ VG.convert col Nothing -> @@ -709,7 +987,7 @@ ( MkTypeErrorContext { userType = Right (typeRep @a) , expectedType = Right (typeRep @b) - , callingFunctionName = Just "toVectorSafe" + , callingFunctionName = Just "toVector" , errorColumnName = Nothing } ) diff --git a/src/DataFrame/Internal/DataFrame.hs b/src/DataFrame/Internal/DataFrame.hs --- a/src/DataFrame/Internal/DataFrame.hs +++ b/src/DataFrame/Internal/DataFrame.hs @@ -19,7 +19,7 @@ import Data.List (sortBy, transpose, (\\)) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import DataFrame.Display.Terminal.PrettyPrint -import DataFrame.Errors (DataFrameException (..)) +import DataFrame.Errors import DataFrame.Internal.Column import Text.Printf import Type.Reflection (typeRep) @@ -129,19 +129,22 @@ All entries in the dataframe must be doubles. This is useful for handing data over into ML systems. -} -toMatrix :: DataFrame -> V.Vector (VU.Vector Float) -toMatrix df = - let - m = V.map (toVector @Double) (columns df) - in - V.generate - (fst (dataframeDimensions df)) - ( \i -> - foldl - (\acc j -> acc `VU.snoc` realToFrac ((m V.! j) V.! i)) - VU.empty - [0 .. (V.length m - 1)] - ) +toMatrix :: DataFrame -> Either DataFrameException (V.Vector (VU.Vector Float)) +toMatrix df = case V.foldl' + (\acc c -> V.snoc <$> acc <*> (toVector @Double c)) + (Right V.empty :: Either DataFrameException (V.Vector (V.Vector Double))) + (columns df) of + Left e -> Left e + Right m -> + pure $ + V.generate + (fst (dataframeDimensions df)) + ( \i -> + foldl + (\acc j -> acc `VU.snoc` realToFrac ((m V.! j) V.! i)) + VU.empty + [0 .. (V.length m - 1)] + ) {- | Get a specific column as a vector. diff --git a/src/DataFrame/Internal/Expression.hs b/src/DataFrame/Internal/Expression.hs --- a/src/DataFrame/Internal/Expression.hs +++ b/src/DataFrame/Internal/Expression.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} @@ -6,6 +7,7 @@ {-# LANGUAGE InstanceSigs #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -13,19 +15,18 @@ module DataFrame.Internal.Expression where -import Control.Exception (throw) import qualified Data.Map as M -import Data.Maybe (fromMaybe) +import Data.Maybe (fromMaybe, isJust) import qualified Data.Text as T import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import qualified Data.Vector as V import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU -import DataFrame.Errors (DataFrameException (ColumnNotFoundException)) +import DataFrame.Errors import DataFrame.Internal.Column import DataFrame.Internal.DataFrame import DataFrame.Internal.Types -import Type.Reflection (Typeable, typeOf, typeRep) +import Type.Reflection (TypeRep, Typeable, typeOf, typeRep, pattern App) data Expr a where Col :: (Columnable a) => T.Text -> Expr a @@ -94,287 +95,771 @@ data UExpr where Wrap :: (Columnable a) => Expr a -> UExpr -interpret :: forall a. (Columnable a) => DataFrame -> Expr a -> TypedColumn a -interpret df (Lit value) = TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value +interpret :: + forall a. + (Columnable a) => + DataFrame -> Expr a -> Either DataFrameException (TypedColumn a) +interpret df (Lit value) = + pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value interpret df (Col name) = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df) - Just col -> TColumn col -interpret df (If cond l r) = - let - (TColumn conditions) = interpret @Bool df cond - (TColumn left) = interpret @a df l - (TColumn right) = interpret @a df r - in - TColumn $ - fromMaybe (error "zipWithColumns returned nothing") $ - zipWithColumns - (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r') - conditions - (zipColumns left right) -interpret df (UnaryOp _ (f :: c -> d) value) = - let - (TColumn value') = interpret @c df value - in - -- TODO: Handle this gracefully. - TColumn $ fromMaybe (error "mapColumn returned nothing") (mapColumn f value') -interpret df (BinaryOp _ (f :: c -> d -> e) (Lit left) (Lit right)) = - TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f left right) -interpret df (BinaryOp _ (f :: c -> d -> e) (Lit left) right) = - let - (TColumn right') = interpret @d df right - in - TColumn $ - fromMaybe (error "mapColumn returned nothing") (mapColumn (f left) right') -interpret df (BinaryOp _ (f :: c -> d -> e) left (Lit right)) = - let - (TColumn left') = interpret @c df left - in - TColumn $ - fromMaybe (error "mapColumn returned nothing") (mapColumn (`f` right) left') -interpret df (BinaryOp _ (f :: c -> d -> e) left right) = - let - (TColumn left') = interpret @c df left - (TColumn right') = interpret @d df right - in + Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df) + Just col -> pure $ TColumn col +interpret df expr@(If cond l r) = case interpret @Bool df cond of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show cond) + } + ) + Left e -> Left e + Right (TColumn conditions) -> case interpret @a df l of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + (context{callingFunctionName = Just "interpret", errorColumnName = Just (show l)}) + Left e -> Left e + Right (TColumn left) -> case interpret @a df r of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + (context{callingFunctionName = Just "interpret", errorColumnName = Just (show r)}) + Left e -> Left e + Right (TColumn right) -> case zipWithColumns + (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r') + conditions + (zipColumns left right) of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right res -> pure $ TColumn res +interpret df expr@(UnaryOp _ (f :: c -> d) value) = case interpret @c df value of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show value) + } + ) + Left e -> Left e + Right (TColumn value') -> case mapColumn f value' of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right res -> pure $ TColumn res +interpret df expr@(BinaryOp _ (f :: c -> d -> e) (Lit left) (Lit right)) = + pure $ TColumn $ - fromMaybe (error "mapColumn returned nothing") (zipWithColumns f left' right') -interpret df (AggVector expr op (f :: v b -> c)) = - let - (TColumn column) = interpret @b df expr - in - case column of - (BoxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of - Just Refl -> interpret @c df (Lit (f col)) - Nothing -> error "Type mismatch" - (OptionalColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of - Just Refl -> interpret @c df (Lit (f col)) - Nothing -> error "Type mismatch" - (UnboxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of - Just Refl -> interpret @c df (Lit (f col)) - Nothing -> error "Type mismatch" -interpret df (AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) = - let - (TColumn column) = interpret @a df expr - in - case headColumn @a column of - Nothing -> error "Invalid operation" - Just h -> case ifoldlColumn (\acc _ v -> f acc v) h column of - Nothing -> error "Invalid operation" - Just value -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value -interpret df (AggNumericVector expr op (f :: VU.Vector b -> c)) = - let - (TColumn column) = interpret @b df expr - in - case column of - (UnboxedColumn (v :: VU.Vector d)) -> case testEquality (typeRep @d) (typeRep @b) of - Just Refl -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f v) - Nothing -> error "Invalid operation" - (BoxedColumn (v :: V.Vector d)) -> case testEquality (typeRep @d) (typeRep @Integer) of - Just Refl -> + fromVector $ + V.replicate (fst $ dataframeDimensions df) (f left right) +interpret df expr@(BinaryOp _ (f :: c -> d -> e) (Lit left) right) = case interpret @d df right of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show right) + } + ) + Left e -> Left e + Right (TColumn right') -> case mapColumn (f left) right' of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right res -> pure $ TColumn res +interpret df expr@(BinaryOp _ (f :: c -> d -> e) left (Lit right)) = case interpret @c df left of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show left) + } + ) + Left e -> Left e + Right (TColumn left') -> case mapColumn (`f` right) left' of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right res -> pure $ TColumn res +interpret df expr@(BinaryOp _ (f :: c -> d -> e) left right) = case interpret @c df left of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show left) + } + ) + Left e -> Left e + Right (TColumn left') -> case interpret @d df right of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show right) + } + ) + Left e -> Left e + Right (TColumn right') -> case zipWithColumns f left' right' of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right res -> pure $ TColumn res +interpret df expression@(AggVector expr op (f :: v b -> c)) = case interpret @b df expr of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right (TColumn column) -> case column of + (BoxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of + Just Refl -> interpret @c df (Lit (f col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @(v b)) + , expectedType = Right (typeOf col) + , callingFunctionName = Just "interpret" + , errorColumnName = Nothing + } + ) + (OptionalColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of + Just Refl -> interpret @c df (Lit (f col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @(v b)) + , expectedType = Right (typeOf col) + , callingFunctionName = Just "interpret" + , errorColumnName = Nothing + } + ) + (UnboxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of + Just Refl -> interpret @c df (Lit (f col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @(v b)) + , expectedType = Right (typeOf col) + , callingFunctionName = Just "interpret" + , errorColumnName = Nothing + } + ) +interpret df expression@(AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) = case interpret @a df expr of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right (TColumn column) -> case headColumn @a column of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left (EmptyDataSetException loc) -> Left (EmptyDataSetException (T.pack $ show expr)) + Left e -> Left e + Right h -> case ifoldlColumn (\acc _ v -> f acc v) h column of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expression) + } + ) + Left e -> Left e + Right value -> + pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value +interpret df expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) = case interpret @b df expr of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right (TColumn column) -> case column of + (UnboxedColumn (v :: VU.Vector d)) -> case testEquality (typeRep @d) (typeRep @b) of + Just Refl -> + pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f v) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpret" + , errorColumnName = Just (show expression) + } + ) + (BoxedColumn (v :: V.Vector d)) -> case testEquality (typeRep @d) (typeRep @Integer) of + Just Refl -> + Right $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f (VU.convert $ V.map fromInteger v)) - Nothing -> error "Invalid operation" - _ -> error "Invalid operation" -interpret df (AggFold expr op start (f :: (a -> b -> a))) = - let - (TColumn column) = interpret @b df expr - in - case ifoldlColumn (\acc _ v -> f acc v) start column of - Nothing -> error "Invalid operation" - Just value -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value - -interpretAggregation :: - forall a. - (Columnable a) => GroupedDataFrame -> Expr a -> Either Column (TypedColumn a) -interpretAggregation gdf (Lit value) = - Right $ TColumn $ fromVector $ V.replicate (VG.length (offsets gdf) - 1) value -interpretAggregation gdf@(Grouped df names indices os) (Col name) = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df) - Just (BoxedColumn col) -> - Left $ - fromVector $ - V.generate - (VU.length os - 1) - ( \i -> - ( V.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - ( \j -> - col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i))) - ) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Integer) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpret" + , errorColumnName = Just (show expression) + } ) - ) - Just (OptionalColumn col) -> - Left $ - fromVector $ - V.generate - (VU.length os - 1) - ( \i -> - ( V.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - ( \j -> - col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i))) + (OptionalColumn (v :: V.Vector (Maybe d))) -> case sNumeric @d of + STrue -> case testEquality (typeRep @d) (typeRep @b) of + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpret" + , errorColumnName = Just (show expression) + } ) + Just Refl -> + pure $ + TColumn $ + fromVector $ + V.replicate + (fst $ dataframeDimensions df) + (f (VU.convert $ V.map (fromMaybe 0) $ V.filter isJust v)) + SFalse -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @d) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "interpret" + , errorColumnName = Just (show expression) + } ) - ) - Just (UnboxedColumn col) -> +interpret df expression@(AggFold expr op start (f :: (a -> b -> a))) = case interpret @b df expr of + Left (TypeMismatchException context) -> Left $ - fromVector $ - V.generate - (VU.length os - 1) - ( \i -> - ( VU.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - ( \j -> - col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i))) - ) - ) + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right (TColumn column) -> case ifoldlColumn (\acc _ v -> f acc v) start column of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpret" + , errorColumnName = Just (show expression) + } ) -interpretAggregation gdf (UnaryOp _ (f :: c -> d) expr) = + Left e -> Left e + Right value -> + pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value + +data AggregationResult a + = UnAggregated Column + | Aggregated (TypedColumn a) + +mkUnaggregatedColumn :: + forall v a. + (VG.Vector v a, Columnable a) => + v a -> VU.Vector Int -> VU.Vector Int -> V.Vector (v a) +mkUnaggregatedColumn col os indices = + V.generate + (VU.length os - 1) + ( \i -> + ( VG.generate + (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) + ( \j -> + col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i))) + ) + ) + ) + +nestedTypeException :: + forall a b. (Typeable a, Typeable b) => String -> DataFrameException +nestedTypeException expression = case typeRep @a of + App t1 t2 -> + TypeMismatchException + ( MkTypeErrorContext + { userType = Left (show (typeRep @b)) :: Either String (TypeRep ()) + , expectedType = Left (show (typeRep @a)) :: Either String (TypeRep ()) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just expression + } + ) + t -> + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @(VU.Vector b)) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just expression + } + ) + +interpretAggregation :: + forall a. + (Columnable a) => + GroupedDataFrame -> Expr a -> Either DataFrameException (AggregationResult a) +interpretAggregation gdf (Lit value) = + Right $ + Aggregated $ + TColumn $ + fromVector $ + V.replicate (VG.length (offsets gdf) - 1) value +interpretAggregation gdf@(Grouped df names indices os) (Col name) = case getColumn name df of + Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df) + Just (BoxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices + Just (OptionalColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices + Just (UnboxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices +interpretAggregation gdf expression@(UnaryOp _ (f :: c -> d) expr) = case interpretAggregation @c gdf expr of - Left unaggregated -> case unaggregated of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expr) + } + ) + Left e -> Left e + Right (UnAggregated unaggregated) -> case unaggregated of BoxedColumn (col :: V.Vector b) -> case testEquality (typeRep @b) (typeRep @(V.Vector c)) of - Just Refl -> Left $ fromVector $ V.map (V.map f) col + Just Refl -> Right $ UnAggregated $ fromVector $ V.map (V.map f) col Nothing -> case testEquality (typeRep @b) (typeRep @(VU.Vector c)) of - Nothing -> error "Type mismatch" + Nothing -> Left $ nestedTypeException @b @c (show expression) Just Refl -> case (sUnbox @c, sUnbox @a) of - (STrue, STrue) -> Left $ fromVector $ V.map (VU.map f) col - (_, _) -> error "" - _ -> error "Very unlikely: unaggregated column is boxed" - Right (TColumn aggregated) -> case mapColumn f aggregated of - Nothing -> error "Type error in interpretation" - Just col -> Right $ TColumn col -interpretAggregation gdf (BinaryOp _ (f :: c -> d -> e) left right) = + (SFalse, _) -> Left $ InternalException "Boxed type inside an unboxed column" + (STrue, STrue) -> Right $ UnAggregated $ fromVector $ V.map (VU.map f) col + (STrue, _) -> Right $ UnAggregated $ fromVector $ V.map (V.map f . VU.convert) col + _ -> Left $ InternalException "Aggregated into a non-boxed column" + Right (Aggregated (TColumn aggregated)) -> case mapColumn f aggregated of + Left e -> Left e + Right col -> Right $ Aggregated $ TColumn col +interpretAggregation gdf expression@(BinaryOp _ (f :: c -> d -> e) left right) = case (interpretAggregation @c gdf left, interpretAggregation @d gdf right) of - (Right (TColumn left'), Right (TColumn right')) -> case zipWithColumns f left' right' of - Nothing -> error "Type error in binary operation" - Just col -> Right $ TColumn col - (Left left', Left right') -> case (left', right') of + (Right (Aggregated (TColumn left')), Right (Aggregated (TColumn right'))) -> case zipWithColumns f left' right' of + Left e -> Left e + Right col -> Right $ Aggregated $ TColumn col + (Right (UnAggregated left'), Right (UnAggregated right')) -> case (left', right') of (BoxedColumn (l :: V.Vector m), BoxedColumn (r :: V.Vector n)) -> case testEquality (typeRep @m) (typeRep @(VU.Vector c)) of Just Refl -> case testEquality (typeRep @n) (typeRep @(VU.Vector d)) of Just Refl -> case (sUnbox @c, sUnbox @d, sUnbox @e) of - (STrue, STrue, STrue) -> Left $ fromVector $ V.zipWith (\l' r' -> VU.zipWith f l' r') l r - (_, _, _) -> error "Weird error" + (STrue, STrue, STrue) -> + Right $ UnAggregated $ fromVector $ V.zipWith (\l' r' -> VU.zipWith f l' r') l r + (STrue, STrue, SFalse) -> + Right $ + UnAggregated $ + fromVector $ + V.zipWith (\l' r' -> V.zipWith f (V.convert l') (V.convert r')) l r + (_, _, _) -> Left $ InternalException "Boxed vectors contain unboxed types" Nothing -> case testEquality (typeRep @n) (typeRep @(V.Vector d)) of Just Refl -> case sUnbox @c of - STrue -> Left $ fromVector $ V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r - SFalse -> error "Very unlikely: boxed value in unboxed column." - Nothing -> error "Not a vector of any type???" - Nothing -> error "Not a vector of any type???" - _ -> error "Very unlikely: aggregation shouldn't be in a boxed column" - (_, _) -> error "Cannot apply binary operation to unaggregated and aggregated column" -interpretAggregation gdf (If cond l r) = + STrue -> + Right $ + UnAggregated $ + fromVector $ + V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r + SFalse -> Left $ InternalException "Unboxed vectors contain boxed types" + Nothing -> Left $ nestedTypeException @n @d (show right) + Nothing -> case testEquality (typeRep @m) (typeRep @(V.Vector c)) of + Nothing -> Left $ nestedTypeException @m @c (show left) + Just Refl -> case testEquality (typeRep @n) (typeRep @(VU.Vector d)) of + Just Refl -> case (sUnbox @d, sUnbox @e) of + (STrue, STrue) -> + Right $ + UnAggregated $ + fromVector $ + V.zipWith + (\l' r' -> V.convert @V.Vector @e @VU.Vector $ V.zipWith f l' (V.convert r')) + l + r + (STrue, SFalse) -> + Right $ + UnAggregated $ + fromVector $ + V.zipWith (\l' r' -> V.zipWith f l' (V.convert r')) l r + (_, _) -> Left $ InternalException "Unboxed vectors contain boxed types" + Nothing -> case testEquality (typeRep @n) (typeRep @(V.Vector d)) of + Just Refl -> case sUnbox @c of + STrue -> case sUnbox @e of + SFalse -> + Right $ + UnAggregated $ + fromVector $ + V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r + STrue -> + Right $ + UnAggregated $ + fromVector $ + V.zipWith (\l' r' -> V.convert @V.Vector @e @VU.Vector $ V.zipWith f l' r') l r + SFalse -> Left $ InternalException "Unboxed vectors contain boxed types" + Nothing -> Left $ nestedTypeException @n @d (show right) + _ -> Left $ InternalException "Aggregated into a non-boxed column" + (Right _, Right _) -> + Left $ + AggregatedAndNonAggregatedException (T.pack $ show left) (T.pack $ show right) + (Left (TypeMismatchException context), _) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show left) + } + ) + (Left e, _) -> Left e + (_, Left (TypeMismatchException context)) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show right) + } + ) + (_, Left e) -> Left e +interpretAggregation gdf expression@(If cond l r) = case ( interpretAggregation @Bool gdf cond , interpretAggregation @a gdf l , interpretAggregation @a gdf r ) of - (Right (TColumn conditions), Right (TColumn left), Right (TColumn right)) -> - Right $ - TColumn $ - fromMaybe (error "zipWithColumns returned nothing") $ - zipWithColumns - (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r') - conditions - (zipColumns left right) - ( Left conditions - , Left left@(BoxedColumn left') - , Left right@(BoxedColumn right') - ) -> - Left $ - fromMaybe (error "zipWithColumns returned nothing") $ - zipWithColumns - ( \(c :: VU.Vector Bool) (l' :: V.Vector a, r' :: V.Vector a) -> - V.zipWith - (\c' (l'', r'') -> if c' then l'' else r'') - (V.convert c) - (V.zip l' r') - ) - conditions - (zipColumns left right) - ( Left conditions - , Left left@(UnboxedColumn left') - , Left right@(UnboxedColumn right') - ) -> case sUnbox @a of - STrue -> + ( Right (Aggregated (TColumn conditions)) + , Right (Aggregated (TColumn left)) + , Right (Aggregated (TColumn right)) + ) -> case zipWithColumns + (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r') + conditions + (zipColumns left right) of + Left e -> Left e + Right v -> Right $ Aggregated (TColumn v) + ( Right (UnAggregated conditions) + , Right (UnAggregated left@(BoxedColumn (left' :: V.Vector b))) + , Right (UnAggregated right@(BoxedColumn (right' :: V.Vector c))) + ) -> case testEquality (typeRep @b) (typeRep @c) of + Nothing -> Left $ - fromMaybe (error "zipWithColumns returned nothing") $ - zipWithColumns - ( \(c :: VU.Vector Bool) (l' :: VU.Vector a, r' :: VU.Vector a) -> VU.zipWith (\c' (l'', r'') -> if c' then l'' else r'') c (VU.zip l' r') + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expression) + } + ) + Just Refl -> case testEquality (typeRep @(V.Vector a)) (typeRep @b) of + Just Refl -> case zipWithColumns + ( \(c :: VU.Vector Bool) (l' :: V.Vector a, r' :: V.Vector a) -> + V.zipWith + (\c' (l'', r'') -> if c' then l'' else r'') + (V.convert c) + (V.zip l' r') + ) + conditions + (zipColumns left right) of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expression) + } + ) + Left e -> Left e + Right v -> Right $ UnAggregated v + Nothing -> case testEquality (typeRep @(VU.Vector a)) (typeRep @b) of + Nothing -> Left $ nestedTypeException @b @a (show expression) + Just Refl -> case sUnbox @a of + SFalse -> Left $ InternalException "Boxed type in unboxed column" + STrue -> case zipWithColumns + ( \(c :: VU.Vector Bool) (l' :: VU.Vector a, r' :: VU.Vector a) -> + VU.zipWith + (\c' (l'', r'') -> if c' then l'' else r'') + c + (VU.zip l' r') ) conditions - (zipColumns left right) - SFalse -> error "Very unlikely: Unboxed type in boxed column" - (_, _, _) -> error "Some columns in the conditional are unaggregated" -interpretAggregation gdf@(Grouped df names indices os) (AggVector expr op (f :: v b -> c)) = + (zipColumns left right) of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expression) + } + ) + Left e -> Left e + Right v -> Right $ UnAggregated v + (Right _, Right _, Right _) -> + Left $ + AggregatedAndNonAggregatedException (T.pack $ show l) (T.pack $ show r) + (Left (TypeMismatchException context), _, _) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show cond) + } + ) + (Left e, _, _) -> Left e + (_, Left (TypeMismatchException context), _) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show l) + } + ) + (_, Left e, _) -> Left e + (_, _, Left (TypeMismatchException context)) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show r) + } + ) + (_, _, Left e) -> Left e +interpretAggregation gdf@(Grouped df names indices os) expression@(AggVector expr op (f :: v b -> c)) = case interpretAggregation @b gdf expr of - Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(v b)) (typeRep @d) of - Nothing -> error "Type mismatch" + Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(v b)) (typeRep @d) of + Nothing -> Left $ nestedTypeException @d @b (show expr) Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of - Nothing -> error "Container mismatch" - Just Refl -> Right $ TColumn $ fromVector $ V.map f col - Left _ -> error "Very unlikely: aggregated into unboxed column???" - Right (TColumn (BoxedColumn col)) -> case testEquality (typeRep @(v b)) (typeOf col) of - Just Refl -> interpretAggregation @c gdf (Lit (f col)) - Nothing -> error "Type mismatch" - Right (TColumn (OptionalColumn col)) -> case testEquality (typeRep @(v b)) (typeOf col) of - Just Refl -> interpretAggregation @c gdf (Lit (f col)) - Nothing -> error "Type mismatch" - Right (TColumn (UnboxedColumn col)) -> case testEquality (typeRep @(v b)) (typeOf col) of + Nothing -> Right $ Aggregated $ TColumn $ fromVector $ V.map (f . V.convert) col + Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map f col + Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column" + Right (Aggregated (TColumn (BoxedColumn (col :: V.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of + Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of + Just Refl -> interpretAggregation @c gdf (Lit (f col)) + Nothing -> interpretAggregation @c gdf (Lit ((f . V.convert) col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expr) + } + ) + Right (Aggregated (TColumn (UnboxedColumn (col :: VU.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of + Just Refl -> case testEquality (typeRep @v) (typeRep @VU.Vector) of + Just Refl -> interpretAggregation @c gdf (Lit (f col)) + Nothing -> interpretAggregation @c gdf (Lit ((f . VU.convert) col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expr) + } + ) + Right (Aggregated (TColumn (OptionalColumn (col :: V.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of + Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of + Just Refl -> interpretAggregation @c gdf (Lit (f col)) + Nothing -> interpretAggregation @c gdf (Lit ((f . V.convert) col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expr) + } + ) + (Left (TypeMismatchException context)) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expression) + } + ) + (Left e) -> Left e +interpretAggregation gdf@(Grouped df names indices os) expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) = + case interpretAggregation @b gdf expr of + (Left (TypeMismatchException context)) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expression) + } + ) + (Left e) -> Left e + Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(VU.Vector b)) (typeRep @d) of + Nothing -> case testEquality (typeRep @(VU.Vector Int)) (typeRep @d) of + Nothing -> case testEquality (typeRep @(V.Vector Integer)) (typeRep @d) of + Nothing -> Left $ nestedTypeException @d @b (show expr) + Just Refl -> + Right $ + Aggregated $ + TColumn $ + fromVector $ + V.map (f . VU.convert . V.map fromIntegral) col + Just Refl -> + Right $ + Aggregated $ + TColumn $ + fromVector $ + V.map f (VG.map (VG.map fromIntegral) col) + Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map f col + Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column" + Right (Aggregated (TColumn (BoxedColumn (col :: V.Vector d)))) -> case testEquality (typeRep @Integer) (typeRep @d) of + Just Refl -> interpretAggregation @c gdf (Lit ((f . V.convert . V.map fromIntegral) col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expr) + } + ) + Right (Aggregated (TColumn (UnboxedColumn (col :: VU.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of Just Refl -> interpretAggregation @c gdf (Lit (f col)) - Nothing -> error "Type mismatch" -interpretAggregation gdf@(Grouped df names indices os) (AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) = + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expr) + } + ) + Right (Aggregated (TColumn (OptionalColumn (col :: V.Vector (Maybe d))))) -> case testEquality (typeRep @b) (typeRep @d) of + Just Refl -> + interpretAggregation @c + gdf + (Lit ((f . V.convert . V.map (fromMaybe 0) . V.filter isJust) col)) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @d) + , callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expr) + } + ) +interpretAggregation gdf@(Grouped df names indices os) expression@(AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) = case interpretAggregation @a gdf expr of - Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(V.Vector a)) (typeRep @d) of + (Left (TypeMismatchException context)) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expression) + } + ) + (Left e) -> Left e + Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(V.Vector a)) (typeRep @d) of Nothing -> case testEquality (typeRep @(VU.Vector a)) (typeRep @d) of - Nothing -> error "Type mismatch" + Nothing -> Left $ nestedTypeException @d @a (show expr) Just Refl -> case sUnbox @a of STrue -> Right $ - TColumn $ - fromVector $ - V.map (\v -> VU.foldl' f (VG.head v) (VG.drop 1 v)) col - SFalse -> error "Very unlikely case: unboxed vector contains a boxed type" + Aggregated $ + TColumn $ + fromVector $ + V.map (\v -> VU.foldl' f (VG.head v) (VG.drop 1 v)) col + SFalse -> Left $ InternalException "Boxed type inside an unboxed column" Just Refl -> Right $ - TColumn $ - fromVector $ - V.map (\v -> VG.foldl' f (VG.head v) (VG.drop 1 v)) col - Left _ -> error "Impossible to aggregate into unaggregated column" - Right (TColumn column) -> case headColumn @a column of - Nothing -> error "Invalid operation" - Just h -> case ifoldlColumn (\acc _ v -> f acc v) h column of - Nothing -> error "Invalid operation" - Just value -> interpretAggregation @a gdf (Lit value) -interpretAggregation gdf@(Grouped df names indices os) (AggFold expr op s (f :: (a -> b -> a))) = + Aggregated $ + TColumn $ + fromVector $ + V.map (\v -> VG.foldl' f (VG.head v) (VG.drop 1 v)) col + Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column" + Right (Aggregated (TColumn column)) -> case headColumn @a column of + Left e -> Left e + Right h -> case ifoldlColumn (\acc _ v -> f acc v) h column of + Left e -> Left e + Right value -> interpretAggregation @a gdf (Lit value) +interpretAggregation gdf@(Grouped df names indices os) expression@(AggFold expr op s (f :: (a -> b -> a))) = case interpretAggregation @b gdf expr of - Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(V.Vector b)) (typeRep @d) of - Just Refl -> Right $ TColumn $ fromVector $ V.map (\v -> V.foldl' f s v) col + (Left (TypeMismatchException context)) -> + Left $ + TypeMismatchException + ( context + { callingFunctionName = Just "interpretAggregation" + , errorColumnName = Just (show expression) + } + ) + (Left e) -> Left e + Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(V.Vector b)) (typeRep @d) of + Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map (\v -> V.foldl' f s v) col Nothing -> case testEquality (typeRep @(VU.Vector b)) (typeRep @d) of Just Refl -> case sUnbox @b of - STrue -> Right $ TColumn $ fromVector $ V.map (\v -> VU.foldl' f s v) col - SFalse -> error "Very unlikely: boxed type in unboxed column" - Nothing -> undefined - Left _ -> error "Impossible to aggregate into unaggregated column" - Right (TColumn column) -> case ifoldlColumn (\acc _ v -> f acc v) s column of - Nothing -> error "Invalid operation" - Just value -> interpretAggregation @a gdf (Lit value) -interpretAggregation gdf@(Grouped df names indices os) (AggNumericVector expr op (f :: VU.Vector b -> c)) = - case interpretAggregation @b gdf expr of - Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(VU.Vector b)) (typeRep @d) of - Nothing -> case testEquality (typeRep @(VU.Vector Int)) (typeRep @d) of - Nothing -> error $ "Type mismatch - non numeric " ++ (show (typeRep @d)) - Just Refl -> Right $ TColumn $ fromVector $ V.map f (VG.map (VG.map fromIntegral) col) - Just Refl -> Right $ TColumn $ fromVector $ V.map f col - Left _ -> error "Impossible to aggregate into unaggregated column" - Right (TColumn (UnboxedColumn col)) -> case testEquality (typeRep @(VU.Vector b)) (typeOf col) of - Just Refl -> interpretAggregation @c gdf (Lit (f col)) - Nothing -> error "Type mismatch" - Right (TColumn (BoxedColumn col)) -> case testEquality (typeRep @(V.Vector Integer)) (typeOf col) of - Just Refl -> interpretAggregation @c gdf (Lit (f (VU.convert $ V.map fromInteger col))) - Nothing -> error "Type mismatch" - Right _ -> error "Cannot apply numeric aggregation to non-numeric column." + STrue -> + Right $ Aggregated $ TColumn $ fromVector $ V.map (\v -> VU.foldl' f s v) col + SFalse -> Left $ InternalException "Boxed type inside an unboxed column" + Nothing -> Left $ nestedTypeException @d @b (show expr) + Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column" + Right (Aggregated (TColumn column)) -> case ifoldlColumn (\acc _ v -> f acc v) s column of + Left e -> Left e + Right value -> interpretAggregation @a gdf (Lit value) instance (Num a, Columnable a) => Num (Expr a) where (+) :: Expr a -> Expr a -> Expr a diff --git a/src/DataFrame/Internal/Parsing.hs b/src/DataFrame/Internal/Parsing.hs --- a/src/DataFrame/Internal/Parsing.hs +++ b/src/DataFrame/Internal/Parsing.hs @@ -41,10 +41,14 @@ {-# INLINE readByteStringInt #-} readByteStringDouble :: (HasCallStack) => C.ByteString -> Maybe Double -readByteStringDouble s = case readSigned readDecimal (C.strip s) of - Nothing -> Nothing - Just (value, "") -> Just value - Just (value, _) -> Nothing +readByteStringDouble s = + let + readFunc = if C.any (\c -> c == 'e' || c == 'E') s then readExponential else readDecimal + in + case readSigned readFunc (C.strip s) of + Nothing -> Nothing + Just (value, "") -> Just value + Just (value, _) -> Nothing {-# INLINE readByteStringDouble #-} readDouble :: (HasCallStack) => T.Text -> Maybe Double diff --git a/src/DataFrame/Internal/Statistics.hs b/src/DataFrame/Internal/Statistics.hs --- a/src/DataFrame/Internal/Statistics.hs +++ b/src/DataFrame/Internal/Statistics.hs @@ -12,7 +12,9 @@ import DataFrame.Errors (DataFrameException (..)) mean' :: (Real a, VU.Unbox a) => VU.Vector a -> Double -mean' samp = VU.sum (VU.map realToFrac samp) / fromIntegral (VU.length samp) +mean' samp + | VU.null samp = throw $ EmptyDataSetException "mean" + | otherwise = VU.sum (VU.map realToFrac samp) / fromIntegral (VU.length samp) {-# INLINE mean' #-} median' :: (Real a, VU.Unbox a) => VU.Vector a -> Double diff --git a/src/DataFrame/Operations/Aggregation.hs b/src/DataFrame/Operations/Aggregation.hs --- a/src/DataFrame/Operations/Aggregation.hs +++ b/src/DataFrame/Operations/Aggregation.hs @@ -18,17 +18,16 @@ import Control.Exception (throw) import Control.Monad.ST (runST) -import Data.Either (fromRight) import Data.Hashable import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) import DataFrame.Errors import DataFrame.Internal.Column ( Column (..), Columnable, + TypedColumn (..), atIndicesStable, getIndices, getIndicesUnboxed, - unwrapTypedColumn, ) import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.Expression @@ -124,9 +123,12 @@ f (name, Wrap (expr :: Expr a)) d = let - value = fromRight (error "Not fully aggregated") (interpretAggregation @a gdf expr) + value = case interpretAggregation @a gdf expr of + Left e -> throw e + Right (UnAggregated _) -> throw $ UnaggregatedException (T.pack $ show expr) + Right (Aggregated (TColumn col)) -> col in - insertColumn name (unwrapTypedColumn value) d + insertColumn name value d in fold f aggs df' diff --git a/src/DataFrame/Operations/Subset.hs b/src/DataFrame/Operations/Subset.hs --- a/src/DataFrame/Operations/Subset.hs +++ b/src/DataFrame/Operations/Subset.hs @@ -29,6 +29,7 @@ import DataFrame.Internal.Expression import DataFrame.Operations.Core import DataFrame.Operations.Transformations (apply) +import System.Random import Type.Reflection import Prelude hiding (filter, take) @@ -163,12 +164,12 @@ filterWhere :: Expr Bool -> DataFrame -> DataFrame filterWhere expr df = let - (TColumn col) = interpret @Bool df expr + (TColumn col) = case interpret @Bool df expr of + Left e -> throw e + Right c -> c indexes = case findIndices (== True) col of - Just ixs -> ixs - Nothing -> - -- Shouldn't happen as we are filtering by Bool column - error "filterWhere: type mismatch" + Right ixs -> ixs + Left e -> throw e c' = snd $ dataframeDimensions df in df @@ -310,3 +311,98 @@ exclude cs df = let keysToKeep = columnNames df L.\\ cs in select keysToKeep df + +{- | Sample a dataframe. The double parameter must be between 0 and 1 (inclusive). + +==== __Example__ +@ +ghci> import System.Random +ghci> D.sample (mkStdGen 137) 0.1 df + +@ +-} +sample :: (RandomGen g) => g -> Double -> DataFrame -> DataFrame +sample pureGen p df = + let + rand = generateRandomVector pureGen (fst (dataframeDimensions df)) + in + df + & insertUnboxedVector "__rand__" rand + & filterWhere (BinaryOp "geq" (>=) (Col @Double "__rand__") (Lit (1 - p))) + & exclude ["__rand__"] + +{- | Split a dataset into two. The first in the tuple gets a sample of p (0 <= p <= 1) and the second gets (1 - p). This is useful for creating test and train splits. + +==== __Example__ +@ +ghci> import System.Random +ghci> D.randomSplit (mkStdGen 137) 0.9 df + +@ +-} +randomSplit :: + (RandomGen g) => g -> Double -> DataFrame -> (DataFrame, DataFrame) +randomSplit pureGen p df = + let + rand = generateRandomVector pureGen (fst (dataframeDimensions df)) + withRand = df & insertUnboxedVector "__rand__" rand + in + ( withRand + & filterWhere (BinaryOp "geq" (>=) (Col @Double "__rand__") (Lit (1 - p))) + & exclude ["__rand__"] + , withRand + & filterWhere (BinaryOp "geq" (<) (Col @Double "__rand__") (Lit p)) + & exclude ["__rand__"] + ) + +{- | Creates n folds of a dataframe. + +==== __Example__ +@ +ghci> import System.Random +ghci> D.kFolds (mkStdGen 137) 5 df + +@ +-} +kFolds :: (RandomGen g) => g -> Int -> DataFrame -> [DataFrame] +kFolds pureGen folds df = + let + rand = generateRandomVector pureGen (fst (dataframeDimensions df)) + withRand = df & insertUnboxedVector "__rand__" rand + partitionSize = 1 / (fromIntegral folds) + singleFold n d = + d + & filterWhere + ( BinaryOp + "geq" + (>=) + (Col @Double "__rand__") + (Lit (fromIntegral n * partitionSize)) + ) + go (-1) _ = [] + go n d = + let + d' = singleFold n d + d'' = + d + & filterWhere + ( BinaryOp + "lt" + (<) + (Col @Double "__rand__") + (Lit (fromIntegral n * partitionSize)) + ) + in + d' : go (n - 1) d'' + in + map (exclude ["__rand__"]) (go (folds - 1) withRand) + +generateRandomVector :: (RandomGen g) => g -> Int -> VU.Vector Double +generateRandomVector pureGen k = VU.fromList $ go pureGen k + where + go g 0 = [] + go g n = + let + (v, g') = uniformR (0 :: Double, 1 :: Double) g + in + v : go g' (n - 1) diff --git a/src/DataFrame/Operations/Transformations.hs b/src/DataFrame/Operations/Transformations.hs --- a/src/DataFrame/Operations/Transformations.hs +++ b/src/DataFrame/Operations/Transformations.hs @@ -17,16 +17,14 @@ import DataFrame.Internal.Column ( Column (..), Columnable, - columnTypeString, + TypedColumn (..), ifoldrColumn, imapColumn, mapColumn, - unwrapTypedColumn, ) import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) import DataFrame.Internal.Expression import DataFrame.Operations.Core -import Type.Reflection (TypeRep, typeRep) -- | O(k) Apply a function to a given column in a dataframe. apply :: @@ -40,6 +38,8 @@ DataFrame -> DataFrame apply f columnName d = case safeApply f columnName d of + Left (TypeMismatchException context) -> + throw $ TypeMismatchException (context{callingFunctionName = Just "apply"}) Left exception -> throw exception Right df -> df @@ -56,28 +56,17 @@ Either DataFrameException DataFrame safeApply f columnName d = case getColumn columnName d of Nothing -> Left $ ColumnNotFoundException columnName "apply" (M.keys $ columnIndices d) - Just column -> case mapColumn f column of - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right $ typeRep @b - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , errorColumnName = Just (T.unpack columnName) - , callingFunctionName = Just "apply" - } - ) - Just column' -> Right $ insertColumn columnName column' d + Just column -> do + column' <- mapColumn f column + pure $ insertColumn columnName column' d {- | O(k) Apply a function to a combination of columns in a dataframe and add the result into `alias` column. -} derive :: forall a. (Columnable a) => T.Text -> Expr a -> DataFrame -> DataFrame -derive name expr df = - let - value = interpret @a df expr - in - insertColumn name (unwrapTypedColumn value) df +derive name expr df = case interpret @a df expr of + Left e -> throw e + Right (TColumn value) -> insertColumn name value df -- | O(k * n) Apply a function to given column names in a dataframe. applyMany :: @@ -142,17 +131,8 @@ (\i val acc -> if condition val then V.cons i acc else acc) V.empty column of - Nothing -> - throw $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right $ typeRep @a - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , errorColumnName = Just (T.unpack columnName) - , callingFunctionName = Just "applyWhere" - } - ) - Just indexes -> + Left e -> throw e + Right indexes -> if V.null indexes then df else L.foldl' (\d i -> applyAtIndex i f columnName d) df indexes @@ -175,17 +155,8 @@ throw $ ColumnNotFoundException columnName "applyAtIndex" (M.keys $ columnIndices df) Just column -> case imapColumn (\index value -> if index == i then f value else value) column of - Nothing -> - throw $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right $ typeRep @a - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , errorColumnName = Just (T.unpack columnName) - , callingFunctionName = Just "applyAtIndex" - } - ) - Just column' -> insertColumn columnName column' df + Left e -> throw e + Right column' -> insertColumn columnName column' df -- | Replace all instances of `Nothing` in a column with the given value. impute :: diff --git a/src/DataFrame/Operations/Typing.hs b/src/DataFrame/Operations/Typing.hs --- a/src/DataFrame/Operations/Typing.hs +++ b/src/DataFrame/Operations/Typing.hs @@ -49,7 +49,9 @@ hasNulls = V.foldl' (\acc v -> if isNothing v then acc || True else acc) False safeVector in - if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c + if safeRead && hasNulls + then BoxedColumn safeVector + else BoxedColumn (V.map T.pack c) Nothing -> BoxedColumn c Just Refl -> let example = T.strip (V.head c) diff --git a/tests/Operations/Apply.hs b/tests/Operations/Apply.hs --- a/tests/Operations/Apply.hs +++ b/tests/Operations/Apply.hs @@ -55,7 +55,16 @@ TestCase ( assertExpectException "[Error Case]" - (DE.typeMismatchError (show $ typeRep @Char) (show $ typeRep @[Char])) + ( show $ + DE.TypeMismatchException + ( DE.MkTypeErrorContext + { DE.userType = Right (typeRep @Char) + , DE.expectedType = Right (typeRep @String) + , DE.callingFunctionName = Just "apply" + , DE.errorColumnName = Nothing + } + ) + ) (print $ DI.getColumn "test2" $ D.apply @Char (const (1 :: Int)) "test2" testData) )