dataframe 0.3.2.0 → 0.3.3.0

raw patch · 16 files changed

+1345/−462 lines, 16 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• CHANGELOG.md +6/−0
• dataframe.cabal +1/−2
• src/DataFrame.hs +3/−0
• src/DataFrame/Errors.hs +19/−17
• src/DataFrame/Functions.hs +31/−10
• src/DataFrame/IO/Parquet.hs +2/−2
• src/DataFrame/Internal/Column.hs +394/−116
• src/DataFrame/Internal/DataFrame.hs +17/−14
• src/DataFrame/Internal/Expression.hs +728/−243
• src/DataFrame/Internal/Parsing.hs +8/−4
• src/DataFrame/Internal/Statistics.hs +3/−1
• src/DataFrame/Operations/Aggregation.hs +6/−4
• src/DataFrame/Operations/Subset.hs +101/−5
• src/DataFrame/Operations/Transformations.hs +13/−42
• src/DataFrame/Operations/Typing.hs +3/−1
• tests/Operations/Apply.hs +10/−1

@@ -1,5 +1,11 @@ # Revision history for dataframe +## 0.3.3.0+* Better error messaging for expression failures.+* Fix bug where exponentials were not being properly during CSV parsing.+* `toMatrix` now returns either an exception or the a vector of vector doubles.+* Add `sample`, `kFolds`, and `randomSplit` to sample dataframes.+ ## 0.3.2.0 * Fix dataframe semigroup instance. Appending two rows of the same name but different types now gives a row of `Either a b` (work by @jhrcek). * Fix left expansion of semigroup instance (work by @jhrcek). 

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.3.2.0+version:            0.3.3.0  synopsis: A fast, safe, and intuitive DataFrame library. @@ -117,7 +117,6 @@                    dataframe ^>= 0.3     default-language: Haskell2010     ghc-options:-      -O2       -threaded       -rtsopts       -with-rtsopts=-N

@@ -278,7 +278,10 @@     filterJust,     filterNothing,     filterWhere,+    kFolds,+    randomSplit,     range,+    sample,     select,     selectBy,     take,

@@ -28,8 +28,12 @@         (Typeable a, Typeable b) =>         TypeErrorContext a b ->         DataFrameException+    AggregatedAndNonAggregatedException :: T.Text -> T.Text -> DataFrameException     ColumnNotFoundException :: T.Text -> T.Text -> [T.Text] -> DataFrameException     EmptyDataSetException :: T.Text -> DataFrameException+    InternalException :: T.Text -> DataFrameException+    NonColumnReferenceException :: T.Text -> DataFrameException+    UnaggregatedException :: T.Text -> DataFrameException     WrongQuantileNumberException :: Int -> DataFrameException     WrongQuantileIndexException :: VU.Vector Int -> Int -> DataFrameException     deriving (Exception)@@ -51,6 +55,14 @@     show (EmptyDataSetException callPoint) = emptyDataSetError callPoint     show (WrongQuantileNumberException q) = wrongQuantileNumberError q     show (WrongQuantileIndexException qs q) = wrongQuantileIndexError qs q+    show (InternalException msg) = "Internal error: " ++ T.unpack msg+    show (NonColumnReferenceException msg) = "Expression must be a column reference in: " ++ T.unpack msg+    show (UnaggregatedException expr) = "Expression is not fully aggregated: " ++ T.unpack expr+    show (AggregatedAndNonAggregatedException expr1 expr2) =+        "Cannot combine aggregated and non-aggregated expressions: \n"+            ++ T.unpack expr1+            ++ "\n"+            ++ T.unpack expr2  columnNotFound :: T.Text -> T.Text -> [T.Text] -> String columnNotFound name callPoint columns =@@ -102,31 +114,21 @@     err         ++ ( "\n\tThis happened when calling function "                 ++ brightGreen cp-                ++ " on the column "+                ++ " on "                 ++ brightGreen name-                ++ "\n\n"-                ++ typeAnnotationSuggestion cp            )-addCallPointInfo Nothing (Just cp) err = err ++ "\n" ++ typeAnnotationSuggestion cp+addCallPointInfo Nothing (Just cp) err =+    err+        ++ ( "\n\tThis happened when calling function "+                ++ brightGreen cp+           ) addCallPointInfo (Just name) Nothing err =     err-        ++ ( "\n\tOn the column "+        ++ ( "\n\tOn "                 ++ name                 ++ "\n\n"            ) addCallPointInfo Nothing Nothing err = err--typeAnnotationSuggestion :: String -> String-typeAnnotationSuggestion cp =-    "\n\n\tTry adding a type at the end of the function e.g "-        ++ "change\n\t\t"-        ++ red (cp ++ " ...")-        ++ " to \n\t\t"-        ++ green ("(" ++ cp ++ " ... :: <Type>)")-        ++ "\n\tor add "-        ++ "{-# LANGUAGE TypeApplications #-} to the top of your "-        ++ "file then change the call to \n\t\t"-        ++ brightGreen (cp ++ " @<Type> ....")  guessColumnName :: T.Text -> [T.Text] -> T.Text guessColumnName userInput columns = case map (\k -> (editDistance userInput k, k)) columns of

@@ -27,6 +27,7 @@ import qualified DataFrame.Operations.Statistics as Stats import DataFrame.Operations.Subset (exclude, select) +import Control.Exception (throw) import Control.Monad import qualified Data.Char as Char import Data.Function@@ -293,7 +294,9 @@         | S.member res seen = go seen xs         | otherwise = (x, res) : go (S.insert res seen) xs       where-        res = interpret df x+        res = case interpret df x of+            Left e -> throw e+            Right v -> v         hasInvalid = case res of             (TColumn (UnboxedColumn (col :: VU.Vector a))) -> case testEquality (typeRep @Double) (typeRep @a) of                 Just Refl -> VU.any (\n -> isNaN n || isInfinite n) col@@ -302,7 +305,9 @@  -- | Checks if two programs generate the same outputs given all the same inputs. equivalent :: DataFrame -> Expr Double -> Expr Double -> Bool-equivalent df p1 p2 = interpret df p1 Prelude.== interpret df p2+equivalent df p1 p2 = case (Prelude.==) <$> interpret df p1 <*> interpret df p2 of+    Left e -> throw e+    Right v -> v  synthesizeFeatureExpr ::     -- | Target expression@@ -316,11 +321,14 @@ synthesizeFeatureExpr target d b df =     let         df' = exclude [target] df+        t = case interpret df (Col target) of+            Left e -> throw e+            Right v -> v      in         case beamSearch             df'             (BeamConfig d b (\l r -> (^ 2) <$> correlation' l r))-            (interpret df (Col target))+            t             [] of             Nothing -> Left "No programs found"             Just p -> Right p@@ -338,6 +346,9 @@     let         df' = exclude [target] df         targetMean = fromMaybe 0 $ Stats.mean target df+        t = case interpret df (Col target) of+            Left e -> throw e+            Right v -> v      in         case beamSearch             df'@@ -351,7 +362,7 @@                         r                 )             )-            (interpret df (Col target))+            t             [] of             Nothing -> Left "No programs found"             Just p ->@@ -362,7 +373,7 @@                                 & select ["_generated_regression_feature_"]                             )                             (BeamConfig d b (\l r -> fmap negate (meanSquaredError l r)))-                            (interpret df (Col target))+                            t                             [Col "_generated_regression_feature_", lit targetMean, lit 10] of                             Nothing -> Left "Could not find coefficients"                             Just p' -> Right (replaceExpr p (Col @Double "_generated_regression_feature_") p')@@ -405,7 +416,9 @@ pickTopN _ _ _ [] = [] pickTopN df (TColumn col) cfg ps =     let-        l = VU.convert (toVector @Double col)+        l = case toVector @Double @VU.Vector col of+            Left e -> throw e+            Right v -> v         ordered =             Prelude.take                 (beamLength cfg)@@ -425,12 +438,18 @@             let                 (TColumn col') = c              in-                VU.convert (toVector @Double col')+                case toVector @Double @VU.Vector col' of+                    Left e -> throw e+                    Right v -> VU.convert v         interpretDoubleVector e =             let-                (TColumn col') = interpret df e+                (TColumn col') = case interpret df e of+                    Left e -> throw e+                    Right v -> v              in-                VU.convert (toVector @Double col')+                case toVector @Double @VU.Vector col' of+                    Left e -> throw e+                    Right v -> VU.convert v      in         trace             ( "Best loss: "@@ -444,7 +463,9 @@ satisfiesExamples :: DataFrame -> TypedColumn Double -> Expr Double -> Bool satisfiesExamples df col expr =     let-        result = interpret df expr+        result = case interpret df expr of+            Left e -> throw e+            Right v -> v      in         result Prelude.== col 

@@ -8,11 +8,11 @@ import Data.Bits import qualified Data.ByteString as BSO import Data.Char+import Data.Either import Data.IORef import Data.Int import qualified Data.List as L import qualified Data.Map as M-import Data.Maybe import qualified Data.Text as T import Data.Word import qualified DataFrame.Internal.Column as DI@@ -272,4 +272,4 @@         [] -> pure $ DI.fromList ([] :: [Maybe Int])         (c : cs) ->             pure $-                L.foldl' (\l r -> fromMaybe (error "concat failed") (DI.concatColumns l r)) c cs+                L.foldl' (\l r -> fromRight (error "concat failed") (DI.concatColumns l r)) c cs

@@ -243,22 +243,49 @@     ) =>     (b -> c) ->     Column ->-    Maybe Column+    Either DataFrameException Column mapColumn f = \case     BoxedColumn (col :: VB.Vector a)         | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Just (fromVector @c (VB.map f col))-        | otherwise -> Nothing+            Right (fromVector @c (VB.map f col))+        | otherwise ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @a)+                        , callingFunctionName = Just "mapColumn"+                        , errorColumnName = Nothing+                        }+                    )     OptionalColumn (col :: VB.Vector a)         | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Just (fromVector @c (VB.map f col))-        | otherwise -> Nothing+            Right (fromVector @c (VB.map f col))+        | otherwise ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @a)+                        , callingFunctionName = Just "mapColumn"+                        , errorColumnName = Nothing+                        }+                    )     UnboxedColumn (col :: VU.Vector a)         | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Just $ case sUnbox @c of+            Right $ case sUnbox @c of                 STrue -> UnboxedColumn (VU.map f col)                 SFalse -> fromVector @c (VB.map f (VB.convert col))-        | otherwise -> Nothing+        | otherwise ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @a)+                        , callingFunctionName = Just "mapColumn"+                        , errorColumnName = Nothing+                        }+                    )  -- | O(1) Gets the number of elements in the column. columnLength :: Column -> Int@@ -322,16 +349,43 @@     (Columnable a) =>     (a -> Bool) ->     Column ->-    Maybe (VU.Vector Int)-findIndices pred (BoxedColumn (column :: VB.Vector b)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    pure $ VG.convert (VG.findIndices pred column)-findIndices pred (UnboxedColumn (column :: VU.Vector b)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    pure $ VG.findIndices pred column-findIndices pred (OptionalColumn (column :: VB.Vector (Maybe b))) = do-    Refl <- testEquality (typeRep @a) (typeRep @(Maybe b))-    pure $ VG.convert (VG.findIndices pred column)+    Either DataFrameException (VU.Vector Int)+findIndices pred (BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl -> pure $ VG.convert (VG.findIndices pred column)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "findIndices"+                    , errorColumnName = Nothing+                    }+                )+findIndices pred (UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl -> pure $ VG.findIndices pred column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "findIndices"+                    , errorColumnName = Nothing+                    }+                )+findIndices pred (OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl -> pure $ VG.convert (VG.findIndices pred column)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "findIndices"+                    , errorColumnName = Nothing+                    }+                )  -- | An internal function that returns a vector of how indexes change after a column is sorted. sortedIndexes :: Bool -> Column -> VU.Vector Int@@ -362,89 +416,271 @@ imapColumn ::     forall b c.     (Columnable b, Columnable c) =>-    (Int -> b -> c) -> Column -> Maybe Column+    (Int -> b -> c) -> Column -> Either DataFrameException Column imapColumn f = \case     BoxedColumn (col :: VB.Vector a)         | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Just (fromVector @c (VB.imap f col))-        | otherwise -> Nothing+            pure (fromVector @c (VB.imap f col))+        | otherwise ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @a)+                        , callingFunctionName = Just "imapColumn"+                        , errorColumnName = Nothing+                        }+                    )     UnboxedColumn (col :: VU.Vector a)         | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Just $+            pure $                 case sUnbox @c of                     STrue -> UnboxedColumn (VU.imap f col)                     SFalse -> fromVector @c (VB.imap f (VB.convert col))-        | otherwise -> Nothing+        | otherwise ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @a)+                        , callingFunctionName = Just "imapColumn"+                        , errorColumnName = Nothing+                        }+                    )     OptionalColumn (col :: VB.Vector a)         | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Just (fromVector @c (VB.imap f col))-        | otherwise -> Nothing+            pure (fromVector @c (VB.imap f col))+        | otherwise ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @a)+                        , callingFunctionName = Just "imapColumn"+                        , errorColumnName = Nothing+                        }+                    )  -- | Filter column with index. ifilterColumn ::-    forall a. (Columnable a) => (Int -> a -> Bool) -> Column -> Maybe Column-ifilterColumn f c@(BoxedColumn (column :: VB.Vector b)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    return $ BoxedColumn $ VG.ifilter f column-ifilterColumn f c@(UnboxedColumn (column :: VU.Vector b)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    return $ UnboxedColumn $ VG.ifilter f column-ifilterColumn _ _ = Nothing+    forall a.+    (Columnable a) =>+    (Int -> a -> Bool) -> Column -> Either DataFrameException Column+ifilterColumn f c@(BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl -> pure $ BoxedColumn $ VG.ifilter f column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "ifilterColumn"+                    , errorColumnName = Nothing+                    }+                )+ifilterColumn f c@(UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl -> pure $ UnboxedColumn $ VG.ifilter f column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "ifilterColumn"+                    , errorColumnName = Nothing+                    }+                )+ifilterColumn f c@(OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl -> pure $ OptionalColumn $ VG.ifilter f column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "ifilterColumn"+                    , errorColumnName = Nothing+                    }+                )  -- | Fold (right) column with index. ifoldrColumn ::     forall a b.-    (Columnable a, Columnable b) => (Int -> a -> b -> b) -> b -> Column -> Maybe b-ifoldrColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = do-    Refl <- testEquality (typeRep @a) (typeRep @d)-    return $ VG.ifoldr f acc column-ifoldrColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = do-    Refl <- testEquality (typeRep @a) (typeRep @d)-    return $ VG.ifoldr f acc column-ifoldrColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = do-    Refl <- testEquality (typeRep @a) (typeRep @d)-    return $ VG.ifoldr f acc column+    (Columnable a, Columnable b) =>+    (Int -> a -> b -> b) -> b -> Column -> Either DataFrameException b+ifoldrColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+    Just Refl -> pure $ VG.ifoldr f acc column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @d)+                    , callingFunctionName = Just "ifoldrColumn"+                    , errorColumnName = Nothing+                    }+                )+ifoldrColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+    Just Refl -> pure $ VG.ifoldr f acc column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @d)+                    , callingFunctionName = Just "ifoldrColumn"+                    , errorColumnName = Nothing+                    }+                )+ifoldrColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+    Just Refl -> pure $ VG.ifoldr f acc column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @d)+                    , callingFunctionName = Just "ifoldrColumn"+                    , errorColumnName = Nothing+                    }+                )  -- | Fold (left) column with index. ifoldlColumn ::     forall a b.-    (Columnable a, Columnable b) => (b -> Int -> a -> b) -> b -> Column -> Maybe b-ifoldlColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = do-    Refl <- testEquality (typeRep @a) (typeRep @d)-    return $ VG.ifoldl' f acc column-ifoldlColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = do-    Refl <- testEquality (typeRep @a) (typeRep @d)-    return $ VG.ifoldl' f acc column-ifoldlColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = do-    Refl <- testEquality (typeRep @a) (typeRep @d)-    return $ VG.ifoldl' f acc column+    (Columnable a, Columnable b) =>+    (b -> Int -> a -> b) -> b -> Column -> Either DataFrameException b+ifoldlColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+    Just Refl -> pure $ VG.ifoldl' f acc column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @d)+                    , callingFunctionName = Just "ifoldlColumn"+                    , errorColumnName = Nothing+                    }+                )+ifoldlColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+    Just Refl -> pure $ VG.ifoldl' f acc column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @d)+                    , callingFunctionName = Just "ifoldlColumn"+                    , errorColumnName = Nothing+                    }+                )+ifoldlColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of+    Just Refl -> pure $ VG.ifoldl' f acc column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @d)+                    , callingFunctionName = Just "ifoldlColumn"+                    , errorColumnName = Nothing+                    }+                ) -headColumn :: forall a. (Columnable a) => Column -> Maybe a-headColumn (BoxedColumn (col :: VB.Vector b)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    pure (VG.head col)-headColumn (UnboxedColumn (col :: VU.Vector b)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    pure (VG.head col)-headColumn (OptionalColumn (col :: VB.Vector b)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    pure (VG.head col)+headColumn :: forall a. (Columnable a) => Column -> Either DataFrameException a+headColumn (BoxedColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl ->+        if VG.null col+            then Left (EmptyDataSetException "headColumn")+            else pure (VG.head col)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "headColumn"+                    , errorColumnName = Nothing+                    }+                )+headColumn (UnboxedColumn (col :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl ->+        if VG.null col+            then Left (EmptyDataSetException "headColumn")+            else pure (VG.head col)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "headColumn"+                    , errorColumnName = Nothing+                    }+                )+headColumn (OptionalColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of+    Just Refl ->+        if VG.null col+            then Left (EmptyDataSetException "headColumn")+            else pure (VG.head col)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "headColumn"+                    , errorColumnName = Nothing+                    }+                )  -- | Generic reduce function for all Column types.-reduceColumn :: forall a b. (Columnable a) => (a -> b) -> Column -> Maybe b+reduceColumn ::+    forall a b.+    (Columnable a, Columnable b) =>+    (a -> b) -> Column -> Either DataFrameException b {-# SPECIALIZE reduceColumn ::-    (VU.Vector (Double, Double) -> Double) -> Column -> Maybe Double-    , (VU.Vector Double -> Double) -> Column -> Maybe Double+    (VU.Vector (Double, Double) -> Double) ->+    Column ->+    Either DataFrameException Double+    , (VU.Vector Double -> Double) -> Column -> Either DataFrameException Double     #-}-reduceColumn f (BoxedColumn (column :: c)) = do-    Refl <- testEquality (typeRep @c) (typeRep @a)-    pure $ f column-reduceColumn f (UnboxedColumn (column :: c)) = do-    Refl <- testEquality (typeRep @c) (typeRep @a)-    pure $ f column-reduceColumn f (OptionalColumn (column :: c)) = do-    Refl <- testEquality (typeRep @c) (typeRep @a)-    pure $ f column+reduceColumn f (BoxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of+    Just Refl -> pure $ f column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "reduceColumn"+                    , errorColumnName = Nothing+                    }+                )+reduceColumn f (UnboxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of+    Just Refl -> pure $ f column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "reduceColumn"+                    , errorColumnName = Nothing+                    }+                )+reduceColumn f (OptionalColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of+    Just Refl -> pure $ f column+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @b)+                    , callingFunctionName = Just "reduceColumn"+                    , errorColumnName = Nothing+                    }+                ) {-# INLINE reduceColumn #-}  -- | An internal, column version of zip.@@ -474,20 +710,43 @@ zipWithColumns ::     forall a b c.     (Columnable a, Columnable b, Columnable c) =>-    (a -> b -> c) -> Column -> Column -> Maybe Column+    (a -> b -> c) -> Column -> Column -> Either DataFrameException Column zipWithColumns f (UnboxedColumn (column :: VU.Vector d)) (UnboxedColumn (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of     Just Refl -> case testEquality (typeRep @b) (typeRep @e) of         Just Refl -> pure $ case sUnbox @c of             STrue -> fromUnboxedVector (VU.zipWith f column other)             SFalse -> fromVector $ VB.zipWith f (VG.convert column) (VG.convert other)-        Nothing -> Nothing-    Nothing -> Nothing-zipWithColumns f left right =-    let-        left' = toVector @a left-        right' = toVector @b right-     in-        pure $ fromVector $ VB.zipWith f left' right'+        Nothing ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @e)+                        , callingFunctionName = Just "zipWithColumns"+                        , errorColumnName = Nothing+                        }+                    )+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeRep @a)+                    , expectedType = Right (typeRep @d)+                    , callingFunctionName = Just "zipWithColumns"+                    , errorColumnName = Nothing+                    }+                )+zipWithColumns f left right = case toVector @a left of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"})+    Left e -> Left e+    Right left' -> case toVector @b right of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"})+        Left e -> Left e+        Right right' -> pure $ fromVector $ VB.zipWith f left' right' {-# INLINE zipWithColumns #-}  -- Functions for mutable columns (intended for IO).@@ -583,17 +842,53 @@ {- | Concatenates two columns. Returns Nothing if the columns are of different types. -}-concatColumns :: Column -> Column -> Maybe Column+concatColumns :: Column -> Column -> Either DataFrameException Column concatColumns (OptionalColumn left) (OptionalColumn right) = case testEquality (typeOf left) (typeOf right) of-    Nothing -> Nothing-    Just Refl -> Just (OptionalColumn $ left <> right)+    Just Refl -> pure (OptionalColumn $ left <> right)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeOf right)+                    , expectedType = Right (typeOf left)+                    , callingFunctionName = Just "concatColumns"+                    , errorColumnName = Nothing+                    }+                ) concatColumns (BoxedColumn left) (BoxedColumn right) = case testEquality (typeOf left) (typeOf right) of-    Nothing -> Nothing-    Just Refl -> Just (BoxedColumn $ left <> right)+    Just Refl -> pure (BoxedColumn $ left <> right)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeOf right)+                    , expectedType = Right (typeOf left)+                    , callingFunctionName = Just "concatColumns"+                    , errorColumnName = Nothing+                    }+                ) concatColumns (UnboxedColumn left) (UnboxedColumn right) = case testEquality (typeOf left) (typeOf right) of-    Nothing -> Nothing-    Just Refl -> Just (UnboxedColumn $ left <> right)-concatColumns _ _ = Nothing+    Just Refl -> pure (UnboxedColumn $ left <> right)+    Nothing ->+        Left $+            TypeMismatchException+                ( MkTypeErrorContext+                    { userType = Right (typeOf right)+                    , expectedType = Right (typeOf left)+                    , callingFunctionName = Just "concatColumns"+                    , errorColumnName = Nothing+                    }+                )+concatColumns left right =+    Left $+        TypeMismatchException+            ( MkTypeErrorContext+                { userType = Right (typeOf right)+                , expectedType = Right (typeOf left)+                , callingFunctionName = Just "concatColumns"+                , errorColumnName = Nothing+                }+            )  {- | Concatenates two columns. @@ -636,23 +931,6 @@     OptionalColumn $         fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right -{- | O(n) Converts a column to a boxed vector. Throws an exception if the wrong type is specified.--__Examples:__--@-> column = fromList [(1 :: Int), 2, 3, 4]-> toVector @Int column-[1,2,3,4]-> toVector @Double column-exception: ...-@--}-toVector :: forall a. (Columnable a) => Column -> VB.Vector a-toVector xs = case toVectorSafe xs of-    Left err -> throw err-    Right val -> val- {- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified.  __Examples:__@@ -666,15 +944,15 @@ @ -} toList :: forall a. (Columnable a) => Column -> [a]-toList xs = case toVectorSafe @a xs of+toList xs = case toVector @a xs of     Left err -> throw err     Right val -> VB.toList val  -- | A safe version of toVector that returns an Either type.-toVectorSafe ::+toVector ::     forall a v.     (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a)-toVectorSafe column@(OptionalColumn (col :: VB.Vector b)) =+toVector column@(OptionalColumn (col :: VB.Vector b)) =     case testEquality (typeRep @a) (typeRep @b) of         Just Refl -> Right $ VG.convert col         Nothing ->@@ -683,11 +961,11 @@                     ( MkTypeErrorContext                         { userType = Right (typeRep @a)                         , expectedType = Right (typeRep @b)-                        , callingFunctionName = Just "toVectorSafe"+                        , callingFunctionName = Just "toVector"                         , errorColumnName = Nothing                         }                     )-toVectorSafe (BoxedColumn (col :: VB.Vector b)) =+toVector (BoxedColumn (col :: VB.Vector b)) =     case testEquality (typeRep @a) (typeRep @b) of         Just Refl -> Right $ VG.convert col         Nothing ->@@ -696,11 +974,11 @@                     ( MkTypeErrorContext                         { userType = Right (typeRep @a)                         , expectedType = Right (typeRep @b)-                        , callingFunctionName = Just "toVectorSafe"+                        , callingFunctionName = Just "toVector"                         , errorColumnName = Nothing                         }                     )-toVectorSafe (UnboxedColumn (col :: VU.Vector b)) =+toVector (UnboxedColumn (col :: VU.Vector b)) =     case testEquality (typeRep @a) (typeRep @b) of         Just Refl -> Right $ VG.convert col         Nothing ->@@ -709,7 +987,7 @@                     ( MkTypeErrorContext                         { userType = Right (typeRep @a)                         , expectedType = Right (typeRep @b)-                        , callingFunctionName = Just "toVectorSafe"+                        , callingFunctionName = Just "toVector"                         , errorColumnName = Nothing                         }                     )

@@ -19,7 +19,7 @@ import Data.List (sortBy, transpose, (\\)) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import DataFrame.Display.Terminal.PrettyPrint-import DataFrame.Errors (DataFrameException (..))+import DataFrame.Errors import DataFrame.Internal.Column import Text.Printf import Type.Reflection (typeRep)@@ -129,19 +129,22 @@ All entries in the dataframe must be doubles. This is useful for handing data over into ML systems. -}-toMatrix :: DataFrame -> V.Vector (VU.Vector Float)-toMatrix df =-    let-        m = V.map (toVector @Double) (columns df)-     in-        V.generate-            (fst (dataframeDimensions df))-            ( \i ->-                foldl-                    (\acc j -> acc `VU.snoc` realToFrac ((m V.! j) V.! i))-                    VU.empty-                    [0 .. (V.length m - 1)]-            )+toMatrix :: DataFrame -> Either DataFrameException (V.Vector (VU.Vector Float))+toMatrix df = case V.foldl'+    (\acc c -> V.snoc <$> acc <*> (toVector @Double c))+    (Right V.empty :: Either DataFrameException (V.Vector (V.Vector Double)))+    (columns df) of+    Left e -> Left e+    Right m ->+        pure $+            V.generate+                (fst (dataframeDimensions df))+                ( \i ->+                    foldl+                        (\acc j -> acc `VU.snoc` realToFrac ((m V.! j) V.! i))+                        VU.empty+                        [0 .. (V.length m - 1)]+                )  {- | Get a specific column as a vector. 

@@ -1,3 +1,4 @@+{-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-}@@ -6,6 +7,7 @@ {-# LANGUAGE InstanceSigs #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}@@ -13,19 +15,18 @@  module DataFrame.Internal.Expression where -import Control.Exception (throw) import qualified Data.Map as M-import Data.Maybe (fromMaybe)+import Data.Maybe (fromMaybe, isJust) import qualified Data.Text as T import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import qualified Data.Vector as V import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU-import DataFrame.Errors (DataFrameException (ColumnNotFoundException))+import DataFrame.Errors import DataFrame.Internal.Column import DataFrame.Internal.DataFrame import DataFrame.Internal.Types-import Type.Reflection (Typeable, typeOf, typeRep)+import Type.Reflection (TypeRep, Typeable, typeOf, typeRep, pattern App)  data Expr a where     Col :: (Columnable a) => T.Text -> Expr a@@ -94,287 +95,771 @@ data UExpr where     Wrap :: (Columnable a) => Expr a -> UExpr -interpret :: forall a. (Columnable a) => DataFrame -> Expr a -> TypedColumn a-interpret df (Lit value) = TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value+interpret ::+    forall a.+    (Columnable a) =>+    DataFrame -> Expr a -> Either DataFrameException (TypedColumn a)+interpret df (Lit value) =+    pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value interpret df (Col name) = case getColumn name df of-    Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df)-    Just col -> TColumn col-interpret df (If cond l r) =-    let-        (TColumn conditions) = interpret @Bool df cond-        (TColumn left) = interpret @a df l-        (TColumn right) = interpret @a df r-     in-        TColumn $-            fromMaybe (error "zipWithColumns returned nothing") $-                zipWithColumns-                    (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r')-                    conditions-                    (zipColumns left right)-interpret df (UnaryOp _ (f :: c -> d) value) =-    let-        (TColumn value') = interpret @c df value-     in-        -- TODO: Handle this gracefully.-        TColumn $ fromMaybe (error "mapColumn returned nothing") (mapColumn f value')-interpret df (BinaryOp _ (f :: c -> d -> e) (Lit left) (Lit right)) =-    TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f left right)-interpret df (BinaryOp _ (f :: c -> d -> e) (Lit left) right) =-    let-        (TColumn right') = interpret @d df right-     in-        TColumn $-            fromMaybe (error "mapColumn returned nothing") (mapColumn (f left) right')-interpret df (BinaryOp _ (f :: c -> d -> e) left (Lit right)) =-    let-        (TColumn left') = interpret @c df left-     in-        TColumn $-            fromMaybe (error "mapColumn returned nothing") (mapColumn (`f` right) left')-interpret df (BinaryOp _ (f :: c -> d -> e) left right) =-    let-        (TColumn left') = interpret @c df left-        (TColumn right') = interpret @d df right-     in+    Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df)+    Just col -> pure $ TColumn col+interpret df expr@(If cond l r) = case interpret @Bool df cond of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show cond)+                    }+                )+    Left e -> Left e+    Right (TColumn conditions) -> case interpret @a df l of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    (context{callingFunctionName = Just "interpret", errorColumnName = Just (show l)})+        Left e -> Left e+        Right (TColumn left) -> case interpret @a df r of+            Left (TypeMismatchException context) ->+                Left $+                    TypeMismatchException+                        (context{callingFunctionName = Just "interpret", errorColumnName = Just (show r)})+            Left e -> Left e+            Right (TColumn right) -> case zipWithColumns+                (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r')+                conditions+                (zipColumns left right) of+                Left (TypeMismatchException context) ->+                    Left $+                        TypeMismatchException+                            ( context+                                { callingFunctionName = Just "interpret"+                                , errorColumnName = Just (show expr)+                                }+                            )+                Left e -> Left e+                Right res -> pure $ TColumn res+interpret df expr@(UnaryOp _ (f :: c -> d) value) = case interpret @c df value of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show value)+                    }+                )+    Left e -> Left e+    Right (TColumn value') -> case mapColumn f value' of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpret"+                        , errorColumnName = Just (show expr)+                        }+                    )+        Left e -> Left e+        Right res -> pure $ TColumn res+interpret df expr@(BinaryOp _ (f :: c -> d -> e) (Lit left) (Lit right)) =+    pure $         TColumn $-            fromMaybe (error "mapColumn returned nothing") (zipWithColumns f left' right')-interpret df (AggVector expr op (f :: v b -> c)) =-    let-        (TColumn column) = interpret @b df expr-     in-        case column of-            (BoxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of-                Just Refl -> interpret @c df (Lit (f col))-                Nothing -> error "Type mismatch"-            (OptionalColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of-                Just Refl -> interpret @c df (Lit (f col))-                Nothing -> error "Type mismatch"-            (UnboxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of-                Just Refl -> interpret @c df (Lit (f col))-                Nothing -> error "Type mismatch"-interpret df (AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) =-    let-        (TColumn column) = interpret @a df expr-     in-        case headColumn @a column of-            Nothing -> error "Invalid operation"-            Just h -> case ifoldlColumn (\acc _ v -> f acc v) h column of-                Nothing -> error "Invalid operation"-                Just value -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value-interpret df (AggNumericVector expr op (f :: VU.Vector b -> c)) =-    let-        (TColumn column) = interpret @b df expr-     in-        case column of-            (UnboxedColumn (v :: VU.Vector d)) -> case testEquality (typeRep @d) (typeRep @b) of-                Just Refl -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f v)-                Nothing -> error "Invalid operation"-            (BoxedColumn (v :: V.Vector d)) -> case testEquality (typeRep @d) (typeRep @Integer) of-                Just Refl ->+            fromVector $+                V.replicate (fst $ dataframeDimensions df) (f left right)+interpret df expr@(BinaryOp _ (f :: c -> d -> e) (Lit left) right) = case interpret @d df right of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show right)+                    }+                )+    Left e -> Left e+    Right (TColumn right') -> case mapColumn (f left) right' of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpret"+                        , errorColumnName = Just (show expr)+                        }+                    )+        Left e -> Left e+        Right res -> pure $ TColumn res+interpret df expr@(BinaryOp _ (f :: c -> d -> e) left (Lit right)) = case interpret @c df left of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show left)+                    }+                )+    Left e -> Left e+    Right (TColumn left') -> case mapColumn (`f` right) left' of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpret"+                        , errorColumnName = Just (show expr)+                        }+                    )+        Left e -> Left e+        Right res -> pure $ TColumn res+interpret df expr@(BinaryOp _ (f :: c -> d -> e) left right) = case interpret @c df left of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show left)+                    }+                )+    Left e -> Left e+    Right (TColumn left') -> case interpret @d df right of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpret"+                        , errorColumnName = Just (show right)+                        }+                    )+        Left e -> Left e+        Right (TColumn right') -> case zipWithColumns f left' right' of+            Left (TypeMismatchException context) ->+                Left $+                    TypeMismatchException+                        ( context+                            { callingFunctionName = Just "interpret"+                            , errorColumnName = Just (show expr)+                            }+                        )+            Left e -> Left e+            Right res -> pure $ TColumn res+interpret df expression@(AggVector expr op (f :: v b -> c)) = case interpret @b df expr of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show expr)+                    }+                )+    Left e -> Left e+    Right (TColumn column) -> case column of+        (BoxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of+            Just Refl -> interpret @c df (Lit (f col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @(v b))+                            , expectedType = Right (typeOf col)+                            , callingFunctionName = Just "interpret"+                            , errorColumnName = Nothing+                            }+                        )+        (OptionalColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of+            Just Refl -> interpret @c df (Lit (f col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @(v b))+                            , expectedType = Right (typeOf col)+                            , callingFunctionName = Just "interpret"+                            , errorColumnName = Nothing+                            }+                        )+        (UnboxedColumn col) -> case testEquality (typeRep @(v b)) (typeOf col) of+            Just Refl -> interpret @c df (Lit (f col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @(v b))+                            , expectedType = Right (typeOf col)+                            , callingFunctionName = Just "interpret"+                            , errorColumnName = Nothing+                            }+                        )+interpret df expression@(AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) = case interpret @a df expr of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show expr)+                    }+                )+    Left e -> Left e+    Right (TColumn column) -> case headColumn @a column of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpret"+                        , errorColumnName = Just (show expr)+                        }+                    )+        Left (EmptyDataSetException loc) -> Left (EmptyDataSetException (T.pack $ show expr))+        Left e -> Left e+        Right h -> case ifoldlColumn (\acc _ v -> f acc v) h column of+            Left (TypeMismatchException context) ->+                Left $+                    TypeMismatchException+                        ( context+                            { callingFunctionName = Just "interpret"+                            , errorColumnName = Just (show expression)+                            }+                        )+            Left e -> Left e+            Right value ->+                pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value+interpret df expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) = case interpret @b df expr of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show expr)+                    }+                )+    Left e -> Left e+    Right (TColumn column) -> case column of+        (UnboxedColumn (v :: VU.Vector d)) -> case testEquality (typeRep @d) (typeRep @b) of+            Just Refl ->+                pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f v)+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @b)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpret"+                            , errorColumnName = Just (show expression)+                            }+                        )+        (BoxedColumn (v :: V.Vector d)) -> case testEquality (typeRep @d) (typeRep @Integer) of+            Just Refl ->+                Right $                     TColumn $                         fromVector $                             V.replicate                                 (fst $ dataframeDimensions df)                                 (f (VU.convert $ V.map fromInteger v))-                Nothing -> error "Invalid operation"-            _ -> error "Invalid operation"-interpret df (AggFold expr op start (f :: (a -> b -> a))) =-    let-        (TColumn column) = interpret @b df expr-     in-        case ifoldlColumn (\acc _ v -> f acc v) start column of-            Nothing -> error "Invalid operation"-            Just value -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value--interpretAggregation ::-    forall a.-    (Columnable a) => GroupedDataFrame -> Expr a -> Either Column (TypedColumn a)-interpretAggregation gdf (Lit value) =-    Right $ TColumn $ fromVector $ V.replicate (VG.length (offsets gdf) - 1) value-interpretAggregation gdf@(Grouped df names indices os) (Col name) = case getColumn name df of-    Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df)-    Just (BoxedColumn col) ->-        Left $-            fromVector $-                V.generate-                    (VU.length os - 1)-                    ( \i ->-                        ( V.generate-                            (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i))-                            ( \j ->-                                col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))-                            )+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Integer)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpret"+                            , errorColumnName = Just (show expression)+                            }                         )-                    )-    Just (OptionalColumn col) ->-        Left $-            fromVector $-                V.generate-                    (VU.length os - 1)-                    ( \i ->-                        ( V.generate-                            (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i))-                            ( \j ->-                                col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))+        (OptionalColumn (v :: V.Vector (Maybe d))) -> case sNumeric @d of+            STrue -> case testEquality (typeRep @d) (typeRep @b) of+                Nothing ->+                    Left $+                        TypeMismatchException+                            ( MkTypeErrorContext+                                { userType = Right (typeRep @b)+                                , expectedType = Right (typeRep @d)+                                , callingFunctionName = Just "interpret"+                                , errorColumnName = Just (show expression)+                                }                             )+                Just Refl ->+                    pure $+                        TColumn $+                            fromVector $+                                V.replicate+                                    (fst $ dataframeDimensions df)+                                    (f (VU.convert $ V.map (fromMaybe 0) $ V.filter isJust v))+            SFalse ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @d)+                            , expectedType = Right (typeRep @b)+                            , callingFunctionName = Just "interpret"+                            , errorColumnName = Just (show expression)+                            }                         )-                    )-    Just (UnboxedColumn col) ->+interpret df expression@(AggFold expr op start (f :: (a -> b -> a))) = case interpret @b df expr of+    Left (TypeMismatchException context) ->         Left $-            fromVector $-                V.generate-                    (VU.length os - 1)-                    ( \i ->-                        ( VU.generate-                            (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i))-                            ( \j ->-                                col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))-                            )-                        )+            TypeMismatchException+                ( context+                    { callingFunctionName = Just "interpret"+                    , errorColumnName = Just (show expr)+                    }+                )+    Left e -> Left e+    Right (TColumn column) -> case ifoldlColumn (\acc _ v -> f acc v) start column of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpret"+                        , errorColumnName = Just (show expression)+                        }                     )-interpretAggregation gdf (UnaryOp _ (f :: c -> d) expr) =+        Left e -> Left e+        Right value ->+            pure $ TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value++data AggregationResult a+    = UnAggregated Column+    | Aggregated (TypedColumn a)++mkUnaggregatedColumn ::+    forall v a.+    (VG.Vector v a, Columnable a) =>+    v a -> VU.Vector Int -> VU.Vector Int -> V.Vector (v a)+mkUnaggregatedColumn col os indices =+    V.generate+        (VU.length os - 1)+        ( \i ->+            ( VG.generate+                (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i))+                ( \j ->+                    col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))+                )+            )+        )++nestedTypeException ::+    forall a b. (Typeable a, Typeable b) => String -> DataFrameException+nestedTypeException expression = case typeRep @a of+    App t1 t2 ->+        TypeMismatchException+            ( MkTypeErrorContext+                { userType = Left (show (typeRep @b)) :: Either String (TypeRep ())+                , expectedType = Left (show (typeRep @a)) :: Either String (TypeRep ())+                , callingFunctionName = Just "interpretAggregation"+                , errorColumnName = Just expression+                }+            )+    t ->+        TypeMismatchException+            ( MkTypeErrorContext+                { userType = Right (typeRep @(VU.Vector b))+                , expectedType = Right (typeRep @b)+                , callingFunctionName = Just "interpretAggregation"+                , errorColumnName = Just expression+                }+            )++interpretAggregation ::+    forall a.+    (Columnable a) =>+    GroupedDataFrame -> Expr a -> Either DataFrameException (AggregationResult a)+interpretAggregation gdf (Lit value) =+    Right $+        Aggregated $+            TColumn $+                fromVector $+                    V.replicate (VG.length (offsets gdf) - 1) value+interpretAggregation gdf@(Grouped df names indices os) (Col name) = case getColumn name df of+    Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df)+    Just (BoxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices+    Just (OptionalColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices+    Just (UnboxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices+interpretAggregation gdf expression@(UnaryOp _ (f :: c -> d) expr) =     case interpretAggregation @c gdf expr of-        Left unaggregated -> case unaggregated of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show expr)+                        }+                    )+        Left e -> Left e+        Right (UnAggregated unaggregated) -> case unaggregated of             BoxedColumn (col :: V.Vector b) -> case testEquality (typeRep @b) (typeRep @(V.Vector c)) of-                Just Refl -> Left $ fromVector $ V.map (V.map f) col+                Just Refl -> Right $ UnAggregated $ fromVector $ V.map (V.map f) col                 Nothing -> case testEquality (typeRep @b) (typeRep @(VU.Vector c)) of-                    Nothing -> error "Type mismatch"+                    Nothing -> Left $ nestedTypeException @b @c (show expression)                     Just Refl -> case (sUnbox @c, sUnbox @a) of-                        (STrue, STrue) -> Left $ fromVector $ V.map (VU.map f) col-                        (_, _) -> error ""-            _ -> error "Very unlikely: unaggregated column is boxed"-        Right (TColumn aggregated) -> case mapColumn f aggregated of-            Nothing -> error "Type error in interpretation"-            Just col -> Right $ TColumn col-interpretAggregation gdf (BinaryOp _ (f :: c -> d -> e) left right) =+                        (SFalse, _) -> Left $ InternalException "Boxed type inside an unboxed column"+                        (STrue, STrue) -> Right $ UnAggregated $ fromVector $ V.map (VU.map f) col+                        (STrue, _) -> Right $ UnAggregated $ fromVector $ V.map (V.map f . VU.convert) col+            _ -> Left $ InternalException "Aggregated into a non-boxed column"+        Right (Aggregated (TColumn aggregated)) -> case mapColumn f aggregated of+            Left e -> Left e+            Right col -> Right $ Aggregated $ TColumn col+interpretAggregation gdf expression@(BinaryOp _ (f :: c -> d -> e) left right) =     case (interpretAggregation @c gdf left, interpretAggregation @d gdf right) of-        (Right (TColumn left'), Right (TColumn right')) -> case zipWithColumns f left' right' of-            Nothing -> error "Type error in binary operation"-            Just col -> Right $ TColumn col-        (Left left', Left right') -> case (left', right') of+        (Right (Aggregated (TColumn left')), Right (Aggregated (TColumn right'))) -> case zipWithColumns f left' right' of+            Left e -> Left e+            Right col -> Right $ Aggregated $ TColumn col+        (Right (UnAggregated left'), Right (UnAggregated right')) -> case (left', right') of             (BoxedColumn (l :: V.Vector m), BoxedColumn (r :: V.Vector n)) -> case testEquality (typeRep @m) (typeRep @(VU.Vector c)) of                 Just Refl -> case testEquality (typeRep @n) (typeRep @(VU.Vector d)) of                     Just Refl -> case (sUnbox @c, sUnbox @d, sUnbox @e) of-                        (STrue, STrue, STrue) -> Left $ fromVector $ V.zipWith (\l' r' -> VU.zipWith f l' r') l r-                        (_, _, _) -> error "Weird error"+                        (STrue, STrue, STrue) ->+                            Right $ UnAggregated $ fromVector $ V.zipWith (\l' r' -> VU.zipWith f l' r') l r+                        (STrue, STrue, SFalse) ->+                            Right $+                                UnAggregated $+                                    fromVector $+                                        V.zipWith (\l' r' -> V.zipWith f (V.convert l') (V.convert r')) l r+                        (_, _, _) -> Left $ InternalException "Boxed vectors contain unboxed types"                     Nothing -> case testEquality (typeRep @n) (typeRep @(V.Vector d)) of                         Just Refl -> case sUnbox @c of-                            STrue -> Left $ fromVector $ V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r-                            SFalse -> error "Very unlikely: boxed value in unboxed column."-                        Nothing -> error "Not a vector of any type???"-                Nothing -> error "Not a vector of any type???"-            _ -> error "Very unlikely: aggregation shouldn't be in a boxed column"-        (_, _) -> error "Cannot apply binary operation to unaggregated and aggregated column"-interpretAggregation gdf (If cond l r) =+                            STrue ->+                                Right $+                                    UnAggregated $+                                        fromVector $+                                            V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r+                            SFalse -> Left $ InternalException "Unboxed vectors contain boxed types"+                        Nothing -> Left $ nestedTypeException @n @d (show right)+                Nothing -> case testEquality (typeRep @m) (typeRep @(V.Vector c)) of+                    Nothing -> Left $ nestedTypeException @m @c (show left)+                    Just Refl -> case testEquality (typeRep @n) (typeRep @(VU.Vector d)) of+                        Just Refl -> case (sUnbox @d, sUnbox @e) of+                            (STrue, STrue) ->+                                Right $+                                    UnAggregated $+                                        fromVector $+                                            V.zipWith+                                                (\l' r' -> V.convert @V.Vector @e @VU.Vector $ V.zipWith f l' (V.convert r'))+                                                l+                                                r+                            (STrue, SFalse) ->+                                Right $+                                    UnAggregated $+                                        fromVector $+                                            V.zipWith (\l' r' -> V.zipWith f l' (V.convert r')) l r+                            (_, _) -> Left $ InternalException "Unboxed vectors contain boxed types"+                        Nothing -> case testEquality (typeRep @n) (typeRep @(V.Vector d)) of+                            Just Refl -> case sUnbox @c of+                                STrue -> case sUnbox @e of+                                    SFalse ->+                                        Right $+                                            UnAggregated $+                                                fromVector $+                                                    V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r+                                    STrue ->+                                        Right $+                                            UnAggregated $+                                                fromVector $+                                                    V.zipWith (\l' r' -> V.convert @V.Vector @e @VU.Vector $ V.zipWith f l' r') l r+                                SFalse -> Left $ InternalException "Unboxed vectors contain boxed types"+                            Nothing -> Left $ nestedTypeException @n @d (show right)+            _ -> Left $ InternalException "Aggregated into a non-boxed column"+        (Right _, Right _) ->+            Left $+                AggregatedAndNonAggregatedException (T.pack $ show left) (T.pack $ show right)+        (Left (TypeMismatchException context), _) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show left)+                        }+                    )+        (Left e, _) -> Left e+        (_, Left (TypeMismatchException context)) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show right)+                        }+                    )+        (_, Left e) -> Left e+interpretAggregation gdf expression@(If cond l r) =     case ( interpretAggregation @Bool gdf cond          , interpretAggregation @a gdf l          , interpretAggregation @a gdf r          ) of-        (Right (TColumn conditions), Right (TColumn left), Right (TColumn right)) ->-            Right $-                TColumn $-                    fromMaybe (error "zipWithColumns returned nothing") $-                        zipWithColumns-                            (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r')-                            conditions-                            (zipColumns left right)-        ( Left conditions-            , Left left@(BoxedColumn left')-            , Left right@(BoxedColumn right')-            ) ->-                Left $-                    fromMaybe (error "zipWithColumns returned nothing") $-                        zipWithColumns-                            ( \(c :: VU.Vector Bool) (l' :: V.Vector a, r' :: V.Vector a) ->-                                V.zipWith-                                    (\c' (l'', r'') -> if c' then l'' else r'')-                                    (V.convert c)-                                    (V.zip l' r')-                            )-                            conditions-                            (zipColumns left right)-        ( Left conditions-            , Left left@(UnboxedColumn left')-            , Left right@(UnboxedColumn right')-            ) -> case sUnbox @a of-                STrue ->+        ( Right (Aggregated (TColumn conditions))+            , Right (Aggregated (TColumn left))+            , Right (Aggregated (TColumn right))+            ) -> case zipWithColumns+                (\(c :: Bool) (l' :: a, r' :: a) -> if c then l' else r')+                conditions+                (zipColumns left right) of+                Left e -> Left e+                Right v -> Right $ Aggregated (TColumn v)+        ( Right (UnAggregated conditions)+            , Right (UnAggregated left@(BoxedColumn (left' :: V.Vector b)))+            , Right (UnAggregated right@(BoxedColumn (right' :: V.Vector c)))+            ) -> case testEquality (typeRep @b) (typeRep @c) of+                Nothing ->                     Left $-                        fromMaybe (error "zipWithColumns returned nothing") $-                            zipWithColumns-                                ( \(c :: VU.Vector Bool) (l' :: VU.Vector a, r' :: VU.Vector a) -> VU.zipWith (\c' (l'', r'') -> if c' then l'' else r'') c (VU.zip l' r')+                        TypeMismatchException+                            ( MkTypeErrorContext+                                { userType = Right (typeRep @b)+                                , expectedType = Right (typeRep @c)+                                , callingFunctionName = Just "interpretAggregation"+                                , errorColumnName = Just (show expression)+                                }+                            )+                Just Refl -> case testEquality (typeRep @(V.Vector a)) (typeRep @b) of+                    Just Refl -> case zipWithColumns+                        ( \(c :: VU.Vector Bool) (l' :: V.Vector a, r' :: V.Vector a) ->+                            V.zipWith+                                (\c' (l'', r'') -> if c' then l'' else r'')+                                (V.convert c)+                                (V.zip l' r')+                        )+                        conditions+                        (zipColumns left right) of+                        Left (TypeMismatchException context) ->+                            Left $+                                TypeMismatchException+                                    ( context+                                        { callingFunctionName = Just "interpretAggregation"+                                        , errorColumnName = Just (show expression)+                                        }+                                    )+                        Left e -> Left e+                        Right v -> Right $ UnAggregated v+                    Nothing -> case testEquality (typeRep @(VU.Vector a)) (typeRep @b) of+                        Nothing -> Left $ nestedTypeException @b @a (show expression)+                        Just Refl -> case sUnbox @a of+                            SFalse -> Left $ InternalException "Boxed type in unboxed column"+                            STrue -> case zipWithColumns+                                ( \(c :: VU.Vector Bool) (l' :: VU.Vector a, r' :: VU.Vector a) ->+                                    VU.zipWith+                                        (\c' (l'', r'') -> if c' then l'' else r'')+                                        c+                                        (VU.zip l' r')                                 )                                 conditions-                                (zipColumns left right)-                SFalse -> error "Very unlikely: Unboxed type in boxed column"-        (_, _, _) -> error "Some columns in the conditional are unaggregated"-interpretAggregation gdf@(Grouped df names indices os) (AggVector expr op (f :: v b -> c)) =+                                (zipColumns left right) of+                                Left (TypeMismatchException context) ->+                                    Left $+                                        TypeMismatchException+                                            ( context+                                                { callingFunctionName = Just "interpretAggregation"+                                                , errorColumnName = Just (show expression)+                                                }+                                            )+                                Left e -> Left e+                                Right v -> Right $ UnAggregated v+        (Right _, Right _, Right _) ->+            Left $+                AggregatedAndNonAggregatedException (T.pack $ show l) (T.pack $ show r)+        (Left (TypeMismatchException context), _, _) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show cond)+                        }+                    )+        (Left e, _, _) -> Left e+        (_, Left (TypeMismatchException context), _) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show l)+                        }+                    )+        (_, Left e, _) -> Left e+        (_, _, Left (TypeMismatchException context)) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show r)+                        }+                    )+        (_, _, Left e) -> Left e+interpretAggregation gdf@(Grouped df names indices os) expression@(AggVector expr op (f :: v b -> c)) =     case interpretAggregation @b gdf expr of-        Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(v b)) (typeRep @d) of-            Nothing -> error "Type mismatch"+        Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(v b)) (typeRep @d) of+            Nothing -> Left $ nestedTypeException @d @b (show expr)             Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of-                Nothing -> error "Container mismatch"-                Just Refl -> Right $ TColumn $ fromVector $ V.map f col-        Left _ -> error "Very unlikely: aggregated into unboxed column???"-        Right (TColumn (BoxedColumn col)) -> case testEquality (typeRep @(v b)) (typeOf col) of-            Just Refl -> interpretAggregation @c gdf (Lit (f col))-            Nothing -> error "Type mismatch"-        Right (TColumn (OptionalColumn col)) -> case testEquality (typeRep @(v b)) (typeOf col) of-            Just Refl -> interpretAggregation @c gdf (Lit (f col))-            Nothing -> error "Type mismatch"-        Right (TColumn (UnboxedColumn col)) -> case testEquality (typeRep @(v b)) (typeOf col) of+                Nothing -> Right $ Aggregated $ TColumn $ fromVector $ V.map (f . V.convert) col+                Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map f col+        Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column"+        Right (Aggregated (TColumn (BoxedColumn (col :: V.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of+            Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of+                Just Refl -> interpretAggregation @c gdf (Lit (f col))+                Nothing -> interpretAggregation @c gdf (Lit ((f . V.convert) col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @b)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpretAggregation"+                            , errorColumnName = Just (show expr)+                            }+                        )+        Right (Aggregated (TColumn (UnboxedColumn (col :: VU.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of+            Just Refl -> case testEquality (typeRep @v) (typeRep @VU.Vector) of+                Just Refl -> interpretAggregation @c gdf (Lit (f col))+                Nothing -> interpretAggregation @c gdf (Lit ((f . VU.convert) col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @b)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpretAggregation"+                            , errorColumnName = Just (show expr)+                            }+                        )+        Right (Aggregated (TColumn (OptionalColumn (col :: V.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of+            Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of+                Just Refl -> interpretAggregation @c gdf (Lit (f col))+                Nothing -> interpretAggregation @c gdf (Lit ((f . V.convert) col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @b)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpretAggregation"+                            , errorColumnName = Just (show expr)+                            }+                        )+        (Left (TypeMismatchException context)) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show expression)+                        }+                    )+        (Left e) -> Left e+interpretAggregation gdf@(Grouped df names indices os) expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) =+    case interpretAggregation @b gdf expr of+        (Left (TypeMismatchException context)) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show expression)+                        }+                    )+        (Left e) -> Left e+        Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(VU.Vector b)) (typeRep @d) of+            Nothing -> case testEquality (typeRep @(VU.Vector Int)) (typeRep @d) of+                Nothing -> case testEquality (typeRep @(V.Vector Integer)) (typeRep @d) of+                    Nothing -> Left $ nestedTypeException @d @b (show expr)+                    Just Refl ->+                        Right $+                            Aggregated $+                                TColumn $+                                    fromVector $+                                        V.map (f . VU.convert . V.map fromIntegral) col+                Just Refl ->+                    Right $+                        Aggregated $+                            TColumn $+                                fromVector $+                                    V.map f (VG.map (VG.map fromIntegral) col)+            Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map f col+        Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column"+        Right (Aggregated (TColumn (BoxedColumn (col :: V.Vector d)))) -> case testEquality (typeRep @Integer) (typeRep @d) of+            Just Refl -> interpretAggregation @c gdf (Lit ((f . V.convert . V.map fromIntegral) col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @b)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpretAggregation"+                            , errorColumnName = Just (show expr)+                            }+                        )+        Right (Aggregated (TColumn (UnboxedColumn (col :: VU.Vector d)))) -> case testEquality (typeRep @b) (typeRep @d) of             Just Refl -> interpretAggregation @c gdf (Lit (f col))-            Nothing -> error "Type mismatch"-interpretAggregation gdf@(Grouped df names indices os) (AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) =+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @b)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpretAggregation"+                            , errorColumnName = Just (show expr)+                            }+                        )+        Right (Aggregated (TColumn (OptionalColumn (col :: V.Vector (Maybe d))))) -> case testEquality (typeRep @b) (typeRep @d) of+            Just Refl ->+                interpretAggregation @c+                    gdf+                    (Lit ((f . V.convert . V.map (fromMaybe 0) . V.filter isJust) col))+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @b)+                            , expectedType = Right (typeRep @d)+                            , callingFunctionName = Just "interpretAggregation"+                            , errorColumnName = Just (show expr)+                            }+                        )+interpretAggregation gdf@(Grouped df names indices os) expression@(AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) =     case interpretAggregation @a gdf expr of-        Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(V.Vector a)) (typeRep @d) of+        (Left (TypeMismatchException context)) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show expression)+                        }+                    )+        (Left e) -> Left e+        Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(V.Vector a)) (typeRep @d) of             Nothing -> case testEquality (typeRep @(VU.Vector a)) (typeRep @d) of-                Nothing -> error "Type mismatch"+                Nothing -> Left $ nestedTypeException @d @a (show expr)                 Just Refl -> case sUnbox @a of                     STrue ->                         Right $-                            TColumn $-                                fromVector $-                                    V.map (\v -> VU.foldl' f (VG.head v) (VG.drop 1 v)) col-                    SFalse -> error "Very unlikely case: unboxed vector contains a boxed type"+                            Aggregated $+                                TColumn $+                                    fromVector $+                                        V.map (\v -> VU.foldl' f (VG.head v) (VG.drop 1 v)) col+                    SFalse -> Left $ InternalException "Boxed type inside an unboxed column"             Just Refl ->                 Right $-                    TColumn $-                        fromVector $-                            V.map (\v -> VG.foldl' f (VG.head v) (VG.drop 1 v)) col-        Left _ -> error "Impossible to aggregate into unaggregated column"-        Right (TColumn column) -> case headColumn @a column of-            Nothing -> error "Invalid operation"-            Just h -> case ifoldlColumn (\acc _ v -> f acc v) h column of-                Nothing -> error "Invalid operation"-                Just value -> interpretAggregation @a gdf (Lit value)-interpretAggregation gdf@(Grouped df names indices os) (AggFold expr op s (f :: (a -> b -> a))) =+                    Aggregated $+                        TColumn $+                            fromVector $+                                V.map (\v -> VG.foldl' f (VG.head v) (VG.drop 1 v)) col+        Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column"+        Right (Aggregated (TColumn column)) -> case headColumn @a column of+            Left e -> Left e+            Right h -> case ifoldlColumn (\acc _ v -> f acc v) h column of+                Left e -> Left e+                Right value -> interpretAggregation @a gdf (Lit value)+interpretAggregation gdf@(Grouped df names indices os) expression@(AggFold expr op s (f :: (a -> b -> a))) =     case interpretAggregation @b gdf expr of-        Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(V.Vector b)) (typeRep @d) of-            Just Refl -> Right $ TColumn $ fromVector $ V.map (\v -> V.foldl' f s v) col+        (Left (TypeMismatchException context)) ->+            Left $+                TypeMismatchException+                    ( context+                        { callingFunctionName = Just "interpretAggregation"+                        , errorColumnName = Just (show expression)+                        }+                    )+        (Left e) -> Left e+        Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(V.Vector b)) (typeRep @d) of+            Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map (\v -> V.foldl' f s v) col             Nothing -> case testEquality (typeRep @(VU.Vector b)) (typeRep @d) of                 Just Refl -> case sUnbox @b of-                    STrue -> Right $ TColumn $ fromVector $ V.map (\v -> VU.foldl' f s v) col-                    SFalse -> error "Very unlikely: boxed type in unboxed column"-                Nothing -> undefined-        Left _ -> error "Impossible to aggregate into unaggregated column"-        Right (TColumn column) -> case ifoldlColumn (\acc _ v -> f acc v) s column of-            Nothing -> error "Invalid operation"-            Just value -> interpretAggregation @a gdf (Lit value)-interpretAggregation gdf@(Grouped df names indices os) (AggNumericVector expr op (f :: VU.Vector b -> c)) =-    case interpretAggregation @b gdf expr of-        Left (BoxedColumn (col :: V.Vector d)) -> case testEquality (typeRep @(VU.Vector b)) (typeRep @d) of-            Nothing -> case testEquality (typeRep @(VU.Vector Int)) (typeRep @d) of-                Nothing -> error $ "Type mismatch - non numeric " ++ (show (typeRep @d))-                Just Refl -> Right $ TColumn $ fromVector $ V.map f (VG.map (VG.map fromIntegral) col)-            Just Refl -> Right $ TColumn $ fromVector $ V.map f col-        Left _ -> error "Impossible to aggregate into unaggregated column"-        Right (TColumn (UnboxedColumn col)) -> case testEquality (typeRep @(VU.Vector b)) (typeOf col) of-            Just Refl -> interpretAggregation @c gdf (Lit (f col))-            Nothing -> error "Type mismatch"-        Right (TColumn (BoxedColumn col)) -> case testEquality (typeRep @(V.Vector Integer)) (typeOf col) of-            Just Refl -> interpretAggregation @c gdf (Lit (f (VU.convert $ V.map fromInteger col)))-            Nothing -> error "Type mismatch"-        Right _ -> error "Cannot apply numeric aggregation to non-numeric column."+                    STrue ->+                        Right $ Aggregated $ TColumn $ fromVector $ V.map (\v -> VU.foldl' f s v) col+                    SFalse -> Left $ InternalException "Boxed type inside an unboxed column"+                Nothing -> Left $ nestedTypeException @d @b (show expr)+        Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column"+        Right (Aggregated (TColumn column)) -> case ifoldlColumn (\acc _ v -> f acc v) s column of+            Left e -> Left e+            Right value -> interpretAggregation @a gdf (Lit value)  instance (Num a, Columnable a) => Num (Expr a) where     (+) :: Expr a -> Expr a -> Expr a

@@ -41,10 +41,14 @@ {-# INLINE readByteStringInt #-}  readByteStringDouble :: (HasCallStack) => C.ByteString -> Maybe Double-readByteStringDouble s = case readSigned readDecimal (C.strip s) of-    Nothing -> Nothing-    Just (value, "") -> Just value-    Just (value, _) -> Nothing+readByteStringDouble s =+    let+        readFunc = if C.any (\c -> c == 'e' || c == 'E') s then readExponential else readDecimal+     in+        case readSigned readFunc (C.strip s) of+            Nothing -> Nothing+            Just (value, "") -> Just value+            Just (value, _) -> Nothing {-# INLINE readByteStringDouble #-}  readDouble :: (HasCallStack) => T.Text -> Maybe Double

@@ -12,7 +12,9 @@ import DataFrame.Errors (DataFrameException (..))  mean' :: (Real a, VU.Unbox a) => VU.Vector a -> Double-mean' samp = VU.sum (VU.map realToFrac samp) / fromIntegral (VU.length samp)+mean' samp+    | VU.null samp = throw $ EmptyDataSetException "mean"+    | otherwise = VU.sum (VU.map realToFrac samp) / fromIntegral (VU.length samp) {-# INLINE mean' #-}  median' :: (Real a, VU.Unbox a) => VU.Vector a -> Double

@@ -18,17 +18,16 @@  import Control.Exception (throw) import Control.Monad.ST (runST)-import Data.Either (fromRight) import Data.Hashable import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) import DataFrame.Errors import DataFrame.Internal.Column (     Column (..),     Columnable,+    TypedColumn (..),     atIndicesStable,     getIndices,     getIndicesUnboxed,-    unwrapTypedColumn,  ) import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.Expression@@ -124,9 +123,12 @@          f (name, Wrap (expr :: Expr a)) d =             let-                value = fromRight (error "Not fully aggregated") (interpretAggregation @a gdf expr)+                value = case interpretAggregation @a gdf expr of+                    Left e -> throw e+                    Right (UnAggregated _) -> throw $ UnaggregatedException (T.pack $ show expr)+                    Right (Aggregated (TColumn col)) -> col              in-                insertColumn name (unwrapTypedColumn value) d+                insertColumn name value d      in         fold f aggs df' 

@@ -29,6 +29,7 @@ import DataFrame.Internal.Expression import DataFrame.Operations.Core import DataFrame.Operations.Transformations (apply)+import System.Random import Type.Reflection import Prelude hiding (filter, take) @@ -163,12 +164,12 @@ filterWhere :: Expr Bool -> DataFrame -> DataFrame filterWhere expr df =     let-        (TColumn col) = interpret @Bool df expr+        (TColumn col) = case interpret @Bool df expr of+            Left e -> throw e+            Right c -> c         indexes = case findIndices (== True) col of-            Just ixs -> ixs-            Nothing ->-                -- Shouldn't happen as we are filtering by Bool column-                error "filterWhere: type mismatch"+            Right ixs -> ixs+            Left e -> throw e         c' = snd $ dataframeDimensions df      in         df@@ -310,3 +311,98 @@ exclude cs df =     let keysToKeep = columnNames df L.\\ cs      in select keysToKeep df++{- | Sample a dataframe. The double parameter must be between 0 and 1 (inclusive).++==== __Example__+@+ghci> import System.Random+ghci> D.sample (mkStdGen 137) 0.1 df++@+-}+sample :: (RandomGen g) => g -> Double -> DataFrame -> DataFrame+sample pureGen p df =+    let+        rand = generateRandomVector pureGen (fst (dataframeDimensions df))+     in+        df+            & insertUnboxedVector "__rand__" rand+            & filterWhere (BinaryOp "geq" (>=) (Col @Double "__rand__") (Lit (1 - p)))+            & exclude ["__rand__"]++{- | Split a dataset into two. The first in the tuple gets a sample of p (0 <= p <= 1) and the second gets (1 - p). This is useful for creating test and train splits.++==== __Example__+@+ghci> import System.Random+ghci> D.randomSplit (mkStdGen 137) 0.9 df++@+-}+randomSplit ::+    (RandomGen g) => g -> Double -> DataFrame -> (DataFrame, DataFrame)+randomSplit pureGen p df =+    let+        rand = generateRandomVector pureGen (fst (dataframeDimensions df))+        withRand = df & insertUnboxedVector "__rand__" rand+     in+        ( withRand+            & filterWhere (BinaryOp "geq" (>=) (Col @Double "__rand__") (Lit (1 - p)))+            & exclude ["__rand__"]+        , withRand+            & filterWhere (BinaryOp "geq" (<) (Col @Double "__rand__") (Lit p))+            & exclude ["__rand__"]+        )++{- | Creates n folds of a dataframe.++==== __Example__+@+ghci> import System.Random+ghci> D.kFolds (mkStdGen 137) 5 df++@+-}+kFolds :: (RandomGen g) => g -> Int -> DataFrame -> [DataFrame]+kFolds pureGen folds df =+    let+        rand = generateRandomVector pureGen (fst (dataframeDimensions df))+        withRand = df & insertUnboxedVector "__rand__" rand+        partitionSize = 1 / (fromIntegral folds)+        singleFold n d =+            d+                & filterWhere+                    ( BinaryOp+                        "geq"+                        (>=)+                        (Col @Double "__rand__")+                        (Lit (fromIntegral n * partitionSize))+                    )+        go (-1) _ = []+        go n d =+            let+                d' = singleFold n d+                d'' =+                    d+                        & filterWhere+                            ( BinaryOp+                                "lt"+                                (<)+                                (Col @Double "__rand__")+                                (Lit (fromIntegral n * partitionSize))+                            )+             in+                d' : go (n - 1) d''+     in+        map (exclude ["__rand__"]) (go (folds - 1) withRand)++generateRandomVector :: (RandomGen g) => g -> Int -> VU.Vector Double+generateRandomVector pureGen k = VU.fromList $ go pureGen k+  where+    go g 0 = []+    go g n =+        let+            (v, g') = uniformR (0 :: Double, 1 :: Double) g+         in+            v : go g' (n - 1)

@@ -17,16 +17,14 @@ import DataFrame.Internal.Column (     Column (..),     Columnable,-    columnTypeString,+    TypedColumn (..),     ifoldrColumn,     imapColumn,     mapColumn,-    unwrapTypedColumn,  ) import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) import DataFrame.Internal.Expression import DataFrame.Operations.Core-import Type.Reflection (TypeRep, typeRep)  -- | O(k) Apply a function to a given column in a dataframe. apply ::@@ -40,6 +38,8 @@     DataFrame ->     DataFrame apply f columnName d = case safeApply f columnName d of+    Left (TypeMismatchException context) ->+        throw $ TypeMismatchException (context{callingFunctionName = Just "apply"})     Left exception -> throw exception     Right df -> df @@ -56,28 +56,17 @@     Either DataFrameException DataFrame safeApply f columnName d = case getColumn columnName d of     Nothing -> Left $ ColumnNotFoundException columnName "apply" (M.keys $ columnIndices d)-    Just column -> case mapColumn f column of-        Nothing ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right $ typeRep @b-                        , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                        , errorColumnName = Just (T.unpack columnName)-                        , callingFunctionName = Just "apply"-                        }-                    )-        Just column' -> Right $ insertColumn columnName column' d+    Just column -> do+        column' <- mapColumn f column+        pure $ insertColumn columnName column' d  {- | O(k) Apply a function to a combination of columns in a dataframe and add the result into `alias` column. -} derive :: forall a. (Columnable a) => T.Text -> Expr a -> DataFrame -> DataFrame-derive name expr df =-    let-        value = interpret @a df expr-     in-        insertColumn name (unwrapTypedColumn value) df+derive name expr df = case interpret @a df expr of+    Left e -> throw e+    Right (TColumn value) -> insertColumn name value df  -- | O(k * n) Apply a function to given column names in a dataframe. applyMany ::@@ -142,17 +131,8 @@         (\i val acc -> if condition val then V.cons i acc else acc)         V.empty         column of-        Nothing ->-            throw $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right $ typeRep @a-                        , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                        , errorColumnName = Just (T.unpack columnName)-                        , callingFunctionName = Just "applyWhere"-                        }-                    )-        Just indexes ->+        Left e -> throw e+        Right indexes ->             if V.null indexes                 then df                 else L.foldl' (\d i -> applyAtIndex i f columnName d) df indexes@@ -175,17 +155,8 @@         throw $             ColumnNotFoundException columnName "applyAtIndex" (M.keys $ columnIndices df)     Just column -> case imapColumn (\index value -> if index == i then f value else value) column of-        Nothing ->-            throw $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right $ typeRep @a-                        , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                        , errorColumnName = Just (T.unpack columnName)-                        , callingFunctionName = Just "applyAtIndex"-                        }-                    )-        Just column' -> insertColumn columnName column' df+        Left e -> throw e+        Right column' -> insertColumn columnName column' df  -- | Replace all instances of `Nothing` in a column with the given value. impute ::

@@ -49,7 +49,9 @@                         hasNulls =                             V.foldl' (\acc v -> if isNothing v then acc || True else acc) False safeVector                      in-                        if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c+                        if safeRead && hasNulls+                            then BoxedColumn safeVector+                            else BoxedColumn (V.map T.pack c)                 Nothing -> BoxedColumn c             Just Refl ->                 let example = T.strip (V.head c)

@@ -55,7 +55,16 @@     TestCase         ( assertExpectException             "[Error Case]"-            (DE.typeMismatchError (show $ typeRep @Char) (show $ typeRep @[Char]))+            ( show $+                DE.TypeMismatchException+                    ( DE.MkTypeErrorContext+                        { DE.userType = Right (typeRep @Char)+                        , DE.expectedType = Right (typeRep @String)+                        , DE.callingFunctionName = Just "apply"+                        , DE.errorColumnName = Nothing+                        }+                    )+            )             (print $ DI.getColumn "test2" $ D.apply @Char (const (1 :: Int)) "test2" testData)         )