dataframe 0.3.3.2 → 0.3.3.3

raw patch · 6 files changed

+429/−13 lines, 6 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• CHANGELOG.md +3/−0
• dataframe.cabal +1/−1
• src/DataFrame.hs +14/−2
• src/DataFrame/Internal/Column.hs +244/−3
• src/DataFrame/Internal/DataFrame.hs +130/−7
• src/DataFrame/Internal/Row.hs +37/−0

@@ -1,5 +1,8 @@ # Revision history for dataframe +## 0.3.3.3+* Split `toMatrix` into more specific `to<Type>Matrix` functions.+ ## 0.3.3.2 * Update documentation on both readthedocs and hackage. 

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.3.3.2+version:            0.3.3.3  synopsis: A fast, safe, and intuitive DataFrame library. 

@@ -251,13 +251,25 @@ import DataFrame.Internal.DataFrame as Dataframe (     DataFrame,     GroupedDataFrame,+    columnAsDoubleVector,+    columnAsFloatVector,+    columnAsIntVector,     columnAsVector,     empty,+    null,+    toDoubleMatrix,+    toFloatMatrix,+    toIntMatrix,     toMarkdownTable,-    toMatrix,  ) import DataFrame.Internal.Expression as Expression (Expr)-import DataFrame.Internal.Row as Row (Row, fromAny, toAny, toRowList)+import DataFrame.Internal.Row as Row (+    Row,+    fromAny,+    toAny,+    toRowList,+    toRowVector,+ ) import DataFrame.Operations.Aggregation as Aggregation (     aggregate,     distinct,

@@ -195,7 +195,7 @@  @ > import qualified Data.Vector as V-> fromVector (V.fromList [(1 :: Int), 2, 3, 4])+> fromVector (VB.fromList [(1 :: Int), 2, 3, 4]) [1,2,3,4] @ -}@@ -211,7 +211,7 @@  @ > import qualified Data.Vector.Unboxed as V-> fromUnboxedVector (V.fromList [(1 :: Int), 2, 3, 4])+> fromUnboxedVector (VB.fromList [(1 :: Int), 2, 3, 4]) [1,2,3,4] @ -}@@ -948,7 +948,35 @@     Left err -> throw err     Right val -> VB.toList val --- | A safe version of toVector that returns an Either type.+{- | Converts a column to a vector of a specific type.++This is a type-safe conversion that requires the column's element type+to exactly match the requested type. You must specify the desired type+via type applications.++==== __Type Parameters__++[@a@] The element type to convert to+[@v@] The vector type (e.g., 'VU.Vector', 'VB.Vector')++==== __Examples__++>>> toVector @Int @VU.Vector column+Right (unboxed vector of Ints)++>>> toVector @Text @VB.Vector column+Right (boxed vector of Text)++==== __Returns__++* 'Right' - The converted vector if types match+* 'Left' 'TypeMismatchException' - If the column's type doesn't match the requested type++==== __See also__++For numeric conversions with automatic type coercion, see 'toDoubleVector',+'toFloatVector', and 'toIntVector'.+-} toVector ::     forall a v.     (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a)@@ -988,6 +1016,219 @@                         { userType = Right (typeRep @a)                         , expectedType = Right (typeRep @b)                         , callingFunctionName = Just "toVector"+                        , errorColumnName = Nothing+                        }+                    )++-- Some common types we will use for numerical computing.++{- | Converts a column to an unboxed vector of 'Double' values.++This function performs intelligent type coercion for numeric types:++* If the column is already 'Double', returns it directly+* If the column contains other floating-point types, converts via 'realToFrac'+* If the column contains integral types, converts via 'fromIntegral' (beware of overflow if the type is `Integer`).++==== __Optional column handling__++For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number).+This allows optional numeric data to be represented in the resulting vector.++==== __Returns__++* 'Right' - The converted 'Double' vector+* 'Left' 'TypeMismatchException' - If the column is not numeric+-}+toDoubleVector :: Column -> Either DataFrameException (VU.Vector Double)+toDoubleVector column =+    case column of+        UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Double) of+            Just Refl -> Right f+            Nothing -> case sFloating @a of+                STrue -> Right (VU.map realToFrac f)+                SFalse -> case sIntegral @a of+                    STrue -> Right (VU.map fromIntegral f)+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Double)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toDoubleVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        OptionalColumn (f :: VB.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @Double) of+            Just Refl -> Right (VB.convert $ VB.map (fromMaybe (read @Double "NaN")) f)+            Nothing -> case sFloating @a of+                STrue ->+                    Right+                        (VB.convert $ VB.map (fromMaybe (read @Double "NaN") . (fmap realToFrac)) f)+                SFalse -> case sIntegral @a of+                    STrue ->+                        Right+                            (VB.convert $ VB.map (fromMaybe (read @Double "NaN") . (fmap fromIntegral)) f)+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Double)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toDoubleVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of+            Just Refl -> Right (VB.convert $ VB.map fromIntegral f)+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Double)+                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                            , callingFunctionName = Just "toDoubleVector"+                            , errorColumnName = Nothing+                            }+                        )++{- | Converts a column to an unboxed vector of 'Float' values.++This function performs intelligent type coercion for numeric types:++* If the column is already 'Float', returns it directly+* If the column contains other floating-point types, converts via 'realToFrac'+* If the column contains integral types, converts via 'fromIntegral'+* If the column is boxed 'Integer', converts via 'fromIntegral' (beware of overflow for 64-bit integers and `Integer`)++==== __Optional column handling__++For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number).+This allows optional numeric data to be represented in the resulting vector.++==== __Returns__++* 'Right' - The converted 'Float' vector+* 'Left' 'TypeMismatchException' - If the column is not numeric++==== __Precision warning__++Converting from 'Double' to 'Float' may result in loss of precision.+-}+toFloatVector :: Column -> Either DataFrameException (VU.Vector Float)+toFloatVector column =+    case column of+        UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Float) of+            Just Refl -> Right f+            Nothing -> case sFloating @a of+                STrue -> Right (VU.map realToFrac f)+                SFalse -> case sIntegral @a of+                    STrue -> Right (VU.map fromIntegral f)+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Float)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toFloatVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        OptionalColumn (f :: VB.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @Float) of+            Just Refl -> Right (VB.convert $ VB.map (fromMaybe (read @Float "NaN")) f)+            Nothing -> case sFloating @a of+                STrue ->+                    Right+                        (VB.convert $ VB.map (fromMaybe (read @Float "NaN") . (fmap realToFrac)) f)+                SFalse -> case sIntegral @a of+                    STrue ->+                        Right+                            (VB.convert $ VB.map (fromMaybe (read @Float "NaN") . (fmap fromIntegral)) f)+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Float)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toFloatVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of+            Just Refl -> Right (VB.convert $ VB.map fromIntegral f)+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Float)+                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                            , callingFunctionName = Just "toFloatVector"+                            , errorColumnName = Nothing+                            }+                        )++{- | Converts a column to an unboxed vector of 'Int' values.++This function performs intelligent type coercion for numeric types:++* If the column is already 'Int', returns it directly+* If the column contains floating-point types, rounds via 'round' and converts+* If the column contains other integral types, converts via 'fromIntegral'+* If the column is boxed 'Integer', converts via 'fromIntegral'++==== __Returns__++* 'Right' - The converted 'Int' vector+* 'Left' 'TypeMismatchException' - If the column is not numeric++==== __Note__++Unlike 'toDoubleVector' and 'toFloatVector', this function does NOT support+'OptionalColumn'. Optional columns must be handled separately.++==== __Rounding behavior__++Floating-point values are rounded to the nearest integer using 'round'.+For example: 2.5 rounds to 2, 3.5 rounds to 4 (banker's rounding).+-}+toIntVector :: Column -> Either DataFrameException (VU.Vector Int)+toIntVector column =+    case column of+        UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of+            Just Refl -> Right f+            Nothing -> case sFloating @a of+                STrue -> Right (VU.map (round . realToFrac) f)+                SFalse -> case sIntegral @a of+                    STrue -> Right (VU.map fromIntegral f)+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Int)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toIntVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of+            Just Refl -> Right (VB.convert $ VB.map fromIntegral f)+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Int)+                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                            , callingFunctionName = Just "toIntVector"+                            , errorColumnName = Nothing+                            }+                        )+        _ ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @Int)+                        , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                        , callingFunctionName = Just "toIntVector"                         , errorColumnName = Nothing                         }                     )

@@ -111,28 +111,59 @@         , dataframeDimensions = (0, 0)         } +{- | Safely retrieves a column by name from the dataframe.++Returns 'Nothing' if the column does not exist.++==== __Examples__++>>> getColumn "age" df+Just (UnboxedColumn ...)++>>> getColumn "nonexistent" df+Nothing+-} getColumn :: T.Text -> DataFrame -> Maybe Column getColumn name df = do     i <- columnIndices df M.!? name     columns df V.!? i +{- | Retrieves a column by name from the dataframe, throwing an exception if not found.++This is an unsafe version of 'getColumn' that throws 'ColumnNotFoundException'+if the column does not exist. Use this when you are certain the column exists.++==== __Throws__++* 'ColumnNotFoundException' - if the column with the given name does not exist+-} unsafeGetColumn :: T.Text -> DataFrame -> Column unsafeGetColumn name df = case getColumn name df of     Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df)     Just col -> col +{- | Checks if the dataframe is empty (has no columns).++Returns 'True' if the dataframe has no columns, 'False' otherwise.+Note that a dataframe with columns but no rows is not considered null.+-} null :: DataFrame -> Bool null df = V.null (columns df) -{- | Returns a dataframe as a two dimentions vector of floats.+{- | Returns a dataframe as a two dimensional vector of floats. -All entries in the dataframe must be doubles.+Converts all columns in the dataframe to float vectors and transposes them+into a row-major matrix representation.+ This is useful for handing data over into ML systems.++Returns 'Left' with an error if any column cannot be converted to floats. -}-toMatrix :: DataFrame -> Either DataFrameException (V.Vector (VU.Vector Float))-toMatrix df = case V.foldl'-    (\acc c -> V.snoc <$> acc <*> (toVector @Double c))-    (Right V.empty :: Either DataFrameException (V.Vector (V.Vector Double)))+toFloatMatrix ::+    DataFrame -> Either DataFrameException (V.Vector (VU.Vector Float))+toFloatMatrix df = case V.foldl'+    (\acc c -> V.snoc <$> acc <*> (toFloatVector c))+    (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Float)))     (columns df) of     Left e -> Left e     Right m ->@@ -141,14 +172,79 @@                 (fst (dataframeDimensions df))                 ( \i ->                     foldl-                        (\acc j -> acc `VU.snoc` realToFrac ((m V.! j) V.! i))+                        (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))                         VU.empty                         [0 .. (V.length m - 1)]                 ) +{- | Returns a dataframe as a two dimensional vector of doubles.++Converts all columns in the dataframe to double vectors and transposes them+into a row-major matrix representation.++This is useful for handing data over into ML systems.++Returns 'Left' with an error if any column cannot be converted to doubles.+-}+toDoubleMatrix ::+    DataFrame -> Either DataFrameException (V.Vector (VU.Vector Double))+toDoubleMatrix df = case V.foldl'+    (\acc c -> V.snoc <$> acc <*> (toDoubleVector c))+    (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Double)))+    (columns df) of+    Left e -> Left e+    Right m ->+        pure $+            V.generate+                (fst (dataframeDimensions df))+                ( \i ->+                    foldl+                        (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))+                        VU.empty+                        [0 .. (V.length m - 1)]+                )++{- | Returns a dataframe as a two dimensional vector of ints.++Converts all columns in the dataframe to int vectors and transposes them+into a row-major matrix representation.++This is useful for handing data over into ML systems.++Returns 'Left' with an error if any column cannot be converted to ints.+-}+toIntMatrix :: DataFrame -> Either DataFrameException (V.Vector (VU.Vector Int))+toIntMatrix df = case V.foldl'+    (\acc c -> V.snoc <$> acc <*> (toIntVector c))+    (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Int)))+    (columns df) of+    Left e -> Left e+    Right m ->+        pure $+            V.generate+                (fst (dataframeDimensions df))+                ( \i ->+                    foldl+                        (\acc j -> acc `VU.snoc` ((m VG.! j) VG.! i))+                        VU.empty+                        [0 .. (V.length m - 1)]+                )+ {- | Get a specific column as a vector.  You must specify the type via type applications.++==== __Examples__++>>> columnAsVector @Int "age" df+[25, 30, 35, ...]++>>> columnAsVector @Text "name" df+["Alice", "Bob", "Charlie", ...]++==== __Throws__++* 'error' - if the column type doesn't match the requested type -} columnAsVector :: forall a. (Columnable a) => T.Text -> DataFrame -> V.Vector a columnAsVector name df = case unsafeGetColumn name df of@@ -161,3 +257,30 @@     (UnboxedColumn (col :: VU.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of         Nothing -> error "Type error"         Just Refl -> VG.convert col++{- | Retrieves a column as an unboxed vector of 'Int' values.++Returns 'Left' with a 'DataFrameException' if the column cannot be converted to ints.+This may occur if the column contains non-numeric data or values outside the 'Int' range.+-}+columnAsIntVector ::+    T.Text -> DataFrame -> Either DataFrameException (VU.Vector Int)+columnAsIntVector name df = toIntVector (unsafeGetColumn name df)++{- | Retrieves a column as an unboxed vector of 'Double' values.++Returns 'Left' with a 'DataFrameException' if the column cannot be converted to doubles.+This may occur if the column contains non-numeric data.+-}+columnAsDoubleVector ::+    T.Text -> DataFrame -> Either DataFrameException (VU.Vector Double)+columnAsDoubleVector name df = toDoubleVector (unsafeGetColumn name df)++{- | Retrieves a column as an unboxed vector of 'Float' values.++Returns 'Left' with a 'DataFrameException' if the column cannot be converted to floats.+This may occur if the column contains non-numeric data.+-}+columnAsFloatVector ::+    T.Text -> DataFrame -> Either DataFrameException (VU.Vector Float)+columnAsFloatVector name df = toFloatVector (unsafeGetColumn name df)

@@ -103,6 +103,22 @@                     Nothing -> acc                     Just Refl -> v' : acc +{- | Converts the entire dataframe to a list of rows.++Each row contains all columns in the dataframe, ordered by their column indices.+The rows are returned in their natural order (from index 0 to n-1).++==== __Examples__++>>> toRowList df+[Row {name = "Alice", age = 25, ...}, Row {name = "Bob", age = 30, ...}, ...]++==== __Performance note__++This function materializes all rows into a list, which may be memory-intensive+for large dataframes. Consider using 'toRowVector' if you need random access+or streaming operations.+-} toRowList :: DataFrame -> [Row] toRowList df =     let@@ -111,6 +127,27 @@      in         map (mkRowRep df nameSet) [0 .. (fst (dataframeDimensions df) - 1)] +{- | Converts the dataframe to a vector of rows with only the specified columns.++Each row will contain only the columns named in the @names@ parameter.+This is useful when you only need a subset of columns or want to control+the column order in the resulting rows.++==== __Parameters__++[@names@] List of column names to include in each row. The order of names+          determines the order of fields in the resulting rows.++[@df@] The dataframe to convert.++==== __Examples__++>>> toRowVector ["name", "age"] df+Vector of rows with only name and age fields++>>> toRowVector [] df  -- Empty column list+Vector of empty rows (one per dataframe row)+-} toRowVector :: [T.Text] -> DataFrame -> V.Vector Row toRowVector names df =     let