dataframe 0.1.0.3 → 0.2.0.0

raw patch · 17 files changed

+389/−287 lines, 17 filesdep ~statisticsPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: statistics

API changes (from Hackage documentation)

Files

• CHANGELOG.md +31/−6
• README.md +5/−0
• app/Main.hs +1/−2
• dataframe.cabal +5/−5
• src/DataFrame.hs +1/−1
• src/DataFrame/Display/Terminal/Plot.hs +2/−2
• src/DataFrame/Errors.hs +2/−2
• src/DataFrame/Internal/Column.hs +178/−126
• src/DataFrame/Internal/Expression.hs +135/−0
• src/DataFrame/Internal/Function.hs +0/−83
• src/DataFrame/Internal/Types.hs +3/−7
• src/DataFrame/Operations/Aggregation.hs +2/−1
• src/DataFrame/Operations/Core.hs +2/−3
• src/DataFrame/Operations/Statistics.hs +1/−1
• src/DataFrame/Operations/Subset.hs +7/−5
• src/DataFrame/Operations/Transformations.hs +8/−37
• tests/Operations/Derive.hs +6/−6

@@ -1,18 +1,43 @@ # Revision history for dataframe -## 0.1.0.0+## 0.2.0.0+### Replace `Function` adt with a column expression syntax. -* Initial release+Previously, we tried to stay as close to Haskell as possible. We used the explicit+ordering of the column names in the first part of the tuple to determine the function+arguments and the a regular Haskell function that we evaluated piece-wise on each row. -## 0.1.0.1+```haskell+let multiply (a :: Int) (b :: Double) = fromIntegral a * b+let withTotalPrice = D.deriveFrom (["quantity", "item_price"], D.func multiply) "total_price" df+``` -* Fixed parse failure on nested, escaped quotation.-* Fixed column info when field name isn't found. +Now, we have a column expression syntax that mirrors Pyspark and Polars. -## 0.1.0.2+```haskell+let withTotalPrice = D.derive "total_price" (D.lift fromIntegral (D.col @Int "quantity") * (D.col @Double"item_price")) df+``` +### Adds a coverage report to the repository (thanks to @oforero)+We don't have good test coverage right now. This will help us determine where to invest.+@oforero provided a script to make an HPC HTML report for coverage.++### Convenience functions for comparisons +Instead of lifting all bool operations we provide `eq`, `leq` etc.++## 0.1.0.3+* Use older version of correlation for ihaskell itegration++## 0.1.0.2 * Change namespace from `Data.DataFrame` to `DataFrame` * Add `toVector` function for converting columns to vectors. * Add `impute` function for replacing `Nothing` values in optional columns. * Add `filterAllJust` to filter out all rows with missing data. * Add `distinct` function that returns a dataframe with distict rows.++## 0.1.0.1+* Fixed parse failure on nested, escaped quotation.+* Fixed column info when field name isn't found.++## 0.1.0.0+* Initial release

@@ -5,11 +5,16 @@ A tool for exploratory data analysis.  ## Installing++### CLI * Install Haskell (ghc + cabal) via [ghcup](https://www.haskell.org/ghcup/install/) selecting all the default options. * To install dataframe run `cabal update && cabal install dataframe` * Open a Haskell repl with dataframe loaded by running `cabal repl --build-depends dataframe`. * Follow along any one of the tutorials below. +### Jupyter notebook+* Use the Dockerfile in the [ihaskell-dataframe](https://github.com/mchav/ihaskell-dataframe) to build and run an image with dataframe integration.+* For a preview check out the [California Housing](https://github.com/mchav/dataframe/blob/main/docs/California%20Housing.ipynb) notebook.  ## What is exploratory data analysis? We provide a primer [here](https://github.com/mchav/dataframe/blob/main/docs/exploratory_data_analysis_primer.md) and show how to do some common analyses.

@@ -101,8 +101,7 @@   print $ D.take 10 f    -- Create a total_price column that is quantity * item_price-  let multiply (a :: Int) (b :: Double) = fromIntegral a * b-  let withTotalPrice = D.deriveFrom (["quantity", "item_price"], D.func multiply) "total_price" f+  let withTotalPrice = D.derive "total_price" (D.lift fromIntegral (D.col @Int "quantity") * D.col @Double"item_price") f    -- sample a filtered subset of the dataframe   putStrLn "Sample dataframe"

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.1.0.3+version:            0.2.0.0  synopsis: An intuitive, dynamically-typed DataFrame library. @@ -24,7 +24,7 @@ library     exposed-modules: DataFrame     other-modules: DataFrame.Internal.Types,-                   DataFrame.Internal.Function,+                   DataFrame.Internal.Expression,                    DataFrame.Internal.Parsing,                    DataFrame.Internal.Column,                    DataFrame.Display.Terminal.PrettyPrint,@@ -48,7 +48,7 @@                       containers >= 0.6.7 && < 0.8,                       directory >= 1.3.0.0 && <= 1.3.9.0,                       hashable >= 1.2 && <= 1.5.0.0,-                      statistics >= 0.16.2.1,+                      statistics >= 0.16.2.1 && <= 0.16.3.0,                       text >= 2.0 && <= 2.1.2,                       time >= 1.12 && <= 1.14,                       vector ^>= 0.13,@@ -60,7 +60,7 @@     main-is:       Main.hs     other-modules: DataFrame,                    DataFrame.Internal.Types,-                   DataFrame.Internal.Function,+                   DataFrame.Internal.Expression,                    DataFrame.Internal.Parsing,                    DataFrame.Internal.Column,                    DataFrame.Display.Terminal.PrettyPrint,@@ -84,7 +84,7 @@                       containers >= 0.6.7 && < 0.8,                       directory >= 1.3.0.0 && <= 1.3.9.0,                       hashable >= 1.2 && <= 1.5.0.0,-                      statistics >= 0.16.2.1,+                      statistics >= 0.16.2.1 && <= 0.16.3.0,                       text >= 2.0 && <= 2.1.2,                       time >= 1.12 && <= 1.14,                       vector ^>= 0.13,

@@ -5,7 +5,7 @@ where  import DataFrame.Internal.Types as D-import DataFrame.Internal.Function as D+import DataFrame.Internal.Expression as D import DataFrame.Internal.Parsing as D import DataFrame.Internal.Column as D import DataFrame.Internal.DataFrame as D hiding (columnIndices, columns)

@@ -5,6 +5,7 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE TupleSections #-}+{-# LANGUAGE FlexibleContexts #-} module DataFrame.Display.Terminal.Plot where  import qualified Data.List as L@@ -19,9 +20,8 @@ import Data.Bifunctor ( first ) import Data.Char ( ord, chr ) import DataFrame.Display.Terminal.Colours-import DataFrame.Internal.Column (Column(..))+import DataFrame.Internal.Column (Column(..), Columnable) import DataFrame.Internal.DataFrame (DataFrame(..))-import DataFrame.Internal.Types (Columnable) import DataFrame.Operations.Core import Data.Maybe (fromMaybe) import Data.Typeable (Typeable)

@@ -22,8 +22,8 @@                           -> T.Text    -- ^ call point                           -> DataFrameException     TypeMismatchException' :: forall a . (Typeable a)-                           => TypeRep a -- ^ expected type-                           -> String    -- ^ given type+                           => TypeRep a -- ^ given type+                           -> String    -- ^ expected type                            -> T.Text    -- ^ column name                            -> T.Text    -- ^ call point                            -> DataFrameException

@@ -1,4 +1,4 @@-{-# LANGUAGE ExplicitNamespaces #-}+{-# LANGUAGE ConstraintKinds #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE InstanceSigs #-} {-# LANGUAGE OverloadedStrings #-}@@ -10,6 +10,11 @@ {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE MultiParamTypeClasses #-}+{-# LANGUAGE DataKinds              #-}+{-# LANGUAGE TypeFamilies           #-}+{-# LANGUAGE PolyKinds #-}+{-# LANGUAGE AllowAmbiguousTypes #-}+{-# LANGUAGE LambdaCase #-} module DataFrame.Internal.Column where  import qualified Data.ByteString.Char8 as C@@ -24,19 +29,20 @@ import qualified Data.Vector.Unboxed.Mutable as VUM  import Control.Monad.ST (runST)-import DataFrame.Internal.Function import DataFrame.Internal.Types import DataFrame.Internal.Parsing import Data.Int import Data.Maybe+import Data.Proxy import Data.Text.Encoding (decodeUtf8Lenient) import Data.Type.Equality (type (:~:)(Refl), TestEquality (..))-import Data.Typeable (Typeable)+import Data.Typeable (Typeable, cast) import Data.Word import Type.Reflection import Unsafe.Coerce (unsafeCoerce) import DataFrame.Errors import Control.Exception (throw)+import Data.Kind (Type, Constraint)  -- | Our representation of a column is a GADT that can store data in either -- a vector with boxed elements or@@ -50,6 +56,12 @@   MutableBoxedColumn :: Columnable a => VBM.IOVector a -> Column   MutableUnboxedColumn :: (Columnable a, VU.Unbox a) => VUM.IOVector a -> Column +data TypedColumn a where+  TColumn :: Columnable a => Column -> TypedColumn a++unwrapTypedColumn :: TypedColumn a -> Column+unwrapTypedColumn (TColumn value) = value+ -- Functions about column metadata. isGrouped :: Column -> Bool isGrouped (GroupedBoxedColumn column) = True@@ -74,6 +86,9 @@   GroupedUnboxedColumn (column :: VB.Vector a) -> show (typeRep @a)   GroupedOptionalColumn (column :: VB.Vector a) -> show (typeRep @a) +instance (Show a) => Show (TypedColumn a) where+  show (TColumn col) = show col+ instance Show Column where   show :: Column -> String   show (BoxedColumn column) = show column@@ -112,48 +127,135 @@       Just Refl -> VB.map (L.sort . VG.toList) a == VB.map (L.sort . VG.toList) b   (==) _ _ = False -class (Columnable a) => Columnify a where-  -- | Converts a boxed vector to a column making sure to put-  -- the vector into an appropriate column type by reflection on the-  -- vector's type parameter.-  toColumn' :: VB.Vector a -> Column+data Rep+  = RBoxed+  | RUnboxed+  | ROptional+  | RGBoxed+  | RGUnboxed+  | RGOptional -instance (Columnable a) => Columnify (Maybe a) where-  toColumn' = OptionalColumn+type family If (cond :: Bool) (yes :: k) (no :: k) :: k where+  If 'True  yes _  = yes+  If 'False _   no = no -instance (Columnable a) => Columnify (VB.Vector a) where-  toColumn' = GroupedBoxedColumn+type family Unboxable (a :: Type) :: Bool where+  Unboxable Int    = 'True+  Unboxable Int8   = 'True+  Unboxable Int16  = 'True+  Unboxable Int32  = 'True+  Unboxable Int64  = 'True+  Unboxable Word   = 'True+  Unboxable Word8  = 'True+  Unboxable Word16 = 'True+  Unboxable Word32 = 'True+  Unboxable Word64 = 'True+  Unboxable Char   = 'True+  Unboxable Bool   = 'True+  Unboxable Double = 'True+  Unboxable Float  = 'True+  Unboxable _      = 'False -instance (Columnable a, VU.Unbox a) => Columnify (VU.Vector a) where-  toColumn' = GroupedUnboxedColumn+-- | Compute the column representation tag for any ‘a’.+type family KindOf a :: Rep where+  KindOf (Maybe a)     = 'ROptional+  KindOf (VB.Vector a) = 'RGBoxed+  KindOf (VU.Vector a) = 'RGUnboxed+  KindOf a             = If (Unboxable a) 'RUnboxed 'RBoxed -instance {-# INCOHERENT #-} (Columnable a) => Columnify a where-  toColumn' xs = case testEquality (typeRep @a) (typeRep @Int) of-    Just Refl -> UnboxedColumn (VU.convert xs)-    Nothing -> case testEquality (typeRep @a) (typeRep @Double) of-      Just Refl -> UnboxedColumn (VU.convert xs)-      Nothing -> case testEquality (typeRep @a) (typeRep @Float) of-        Just Refl -> UnboxedColumn (VU.convert xs)-        Nothing -> BoxedColumn xs+class ColumnifyRep (r :: Rep) a where+  toColumnRep :: VB.Vector a -> Column -class (Columnable a) => ColumnifyList a where-  -- | Converts a boxed vector to a column making sure to put-  -- the vector into an appropriate column type by reflection on the-  -- vector's type parameter.-  toColumn :: [a] -> Column+type Columnable a = (Columnable' a, ColumnifyRep (KindOf a) a, UnboxIf a, SBoolI (Unboxable a) ) -instance (Columnable a) => ColumnifyList (Maybe a) where-  toColumn = OptionalColumn . VB.fromList+instance (Columnable a, VU.Unbox a)+      => ColumnifyRep 'RUnboxed a where+  toColumnRep = UnboxedColumn . VU.convert -instance {-# INCOHERENT #-} (Columnable a) => ColumnifyList a where-  toColumn xs = case testEquality (typeRep @a) (typeRep @Int) of-    Just Refl -> UnboxedColumn (VU.fromList xs)-    Nothing -> case testEquality (typeRep @a) (typeRep @Double) of-      Just Refl -> UnboxedColumn (VU.fromList xs)-      Nothing -> case testEquality (typeRep @a) (typeRep @Float) of-        Just Refl -> UnboxedColumn (VU.fromList xs)-        Nothing -> BoxedColumn (VB.fromList xs)+instance Columnable a+      => ColumnifyRep 'RBoxed a where+  toColumnRep = BoxedColumn +instance Columnable a+      => ColumnifyRep 'ROptional (Maybe a) where+  toColumnRep = OptionalColumn++instance Columnable a+      => ColumnifyRep 'RGBoxed (VB.Vector a) where+  toColumnRep = GroupedBoxedColumn++instance (Columnable a, VU.Unbox a)+      => ColumnifyRep 'RGUnboxed (VU.Vector a) where+  toColumnRep = GroupedUnboxedColumn++toColumn' ::+  forall a. (Columnable a, ColumnifyRep (KindOf a) a)+  => VB.Vector a -> Column+toColumn' = toColumnRep @(KindOf a)++toColumn ::+  forall a. (Columnable a, ColumnifyRep (KindOf a) a)+  => [a] -> Column+toColumn = toColumnRep @(KindOf a) . VB.fromList++data SBool (b :: Bool) where+  STrue  :: SBool 'True+  SFalse :: SBool 'False++class SBoolI (b :: Bool) where+  sbool :: SBool b          -- the run-time witness++instance SBoolI 'True  where sbool = STrue+instance SBoolI 'False where sbool = SFalse++sUnbox :: forall a. SBoolI (Unboxable a) => SBool (Unboxable a)+sUnbox = sbool @(Unboxable a)++type family When (flag :: Bool) (c :: Constraint) :: Constraint where+  When 'True  c = c+  When 'False c = ()          -- empty constraint++type UnboxIf a = When (Unboxable a) (VU.Unbox a)++-- | Generic column transformation (no index).+transform+  :: forall b c.+     ( Columnable b+     , Columnable c+     , UnboxIf c+     , Typeable b+     , Typeable c )+  => (b -> c)+  -> Column+  -> Maybe Column+transform f = \case+  BoxedColumn (col :: VB.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just (toColumn' @c (VB.map f col))+    | otherwise -> Nothing+  OptionalColumn (col :: VB.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just (toColumn' @c (VB.map f col))+    | otherwise -> Nothing+  UnboxedColumn (col :: VU.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just $ case sUnbox @c of+                STrue  -> UnboxedColumn (VU.map f col)+                SFalse -> toColumn' @c (VB.map f (VB.convert col))+    | otherwise -> Nothing+  GroupedBoxedColumn (col :: VB.Vector (VB.Vector a))+    | Just Refl <- testEquality (typeRep @(VB.Vector a)) (typeRep @b)+    -> Just (toColumn' @c (VB.map f col))+    | otherwise -> Nothing+  GroupedUnboxedColumn (col :: VB.Vector (VU.Vector a))+    | Just Refl <- testEquality (typeRep @(VU.Vector a)) (typeRep @b)+    -> Just (toColumn' @c (VB.map f col))+    | otherwise -> Nothing+  GroupedOptionalColumn (col :: VB.Vector (VB.Vector a))+    | Just Refl <- testEquality (typeRep @(VB.Vector a)) (typeRep @b)+    -> Just (toColumn' @c (VB.map f col))+    | otherwise -> Nothing+ -- | Converts a an unboxed vector to a column making sure to put -- the vector into an appropriate column type by reflection on the -- vector's type parameter.@@ -255,79 +357,34 @@   return $ VU.generate (VG.length column) (\i -> fst (sorted VG.! i)) {-# INLINE sortedIndexes #-} --- Operations on a column that may change its type.--instance Transformable Column where-  transform :: forall b c . (Columnable b, Columnable c) => (b -> c) -> Column -> Maybe Column-  transform f (BoxedColumn (column :: VB.Vector a)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    return (toColumn' (VB.map f column))-  transform f (OptionalColumn (column :: VB.Vector a)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    return (toColumn' (VB.map f column))-  transform f (UnboxedColumn (column :: VU.Vector a)) = do-    Refl <- testEquality (typeRep @a) (typeRep @b)-    return $ if testUnboxable (typeRep @c) then transformUnboxed f column else toColumn' (VB.map f (VB.convert column))-  transform f (GroupedBoxedColumn (column :: VB.Vector (VB.Vector a))) = do-    Refl <- testEquality (typeRep @(VB.Vector a)) (typeRep @b)-    return (toColumn' (VB.map f column))-  transform f (GroupedUnboxedColumn (column :: VB.Vector (VU.Vector a))) = do-    Refl <- testEquality (typeRep @(VU.Vector a)) (typeRep @b)-    return (toColumn' (VB.map f column))-  transform f (GroupedOptionalColumn (column :: VB.Vector (VB.Vector a))) = do-    Refl <- testEquality (typeRep @(VB.Vector a)) (typeRep @b)-    return (toColumn' (VB.map f column))- -- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column.-transformUnboxed :: forall a b . (Columnable a, VU.Unbox a, Columnable b) => (a -> b) -> VU.Vector a -> Column-transformUnboxed f = itransformUnboxed (const f)---- TODO: Make a type class with incoherent instances.-itransformUnboxed :: forall a b . (Columnable a, VU.Unbox a, Columnable b) => (Int -> a -> b) -> VU.Vector a -> Column-itransformUnboxed f column = case testEquality (typeRep @b) (typeRep @Int) of-  Just Refl -> UnboxedColumn $ VU.imap f column-  Nothing -> case testEquality (typeRep @b) (typeRep @Int8) of-    Just Refl -> UnboxedColumn $ VU.imap f column-    Nothing -> case testEquality (typeRep @b) (typeRep @Int16) of-      Just Refl -> UnboxedColumn $ VU.imap f column-      Nothing -> case testEquality (typeRep @b) (typeRep @Int32) of-        Just Refl -> UnboxedColumn $ VU.imap f column-        Nothing -> case testEquality (typeRep @b) (typeRep @Int64) of-          Just Refl -> UnboxedColumn $ VU.imap f column-          Nothing -> case testEquality (typeRep @b) (typeRep @Word8) of-            Just Refl -> UnboxedColumn $ VU.imap f column-            Nothing-> case testEquality (typeRep @b) (typeRep @Word16) of-              Just Refl -> UnboxedColumn $ VU.imap f column-              Nothing -> case testEquality (typeRep @b) (typeRep @Word32) of-                Just Refl -> UnboxedColumn $ VU.imap f column-                Nothing -> case testEquality (typeRep @b) (typeRep @Word64) of-                  Just Refl -> UnboxedColumn $ VU.imap f column-                  Nothing -> case testEquality (typeRep @b) (typeRep @Char) of-                    Just Refl -> UnboxedColumn $ VU.imap f column-                    Nothing -> case testEquality (typeRep @b) (typeRep @Bool) of-                      Just Refl -> UnboxedColumn $ VU.imap f column-                      Nothing -> case testEquality (typeRep @b) (typeRep @Float) of-                        Just Refl -> UnboxedColumn $ VU.imap f column-                        Nothing -> case testEquality (typeRep @b) (typeRep @Double) of-                          Just Refl -> UnboxedColumn $ VU.imap f column-                          Nothing -> case testEquality (typeRep @b) (typeRep @Word) of-                            Just Refl -> UnboxedColumn $ VU.imap f column-                            Nothing -> error "Result type is unboxed" -- since we only call this after confirming ---- | tranform with index.-itransform :: forall b c. (Columnable b, Columnable c) => (Int -> b -> c) -> Column -> Maybe Column-itransform f (BoxedColumn (column :: VB.Vector a)) = do-  Refl <- testEquality (typeRep @a) (typeRep @b)-  return (toColumn' (VB.imap f column))-itransform f (UnboxedColumn (column :: VU.Vector a)) = do-  Refl <- testEquality (typeRep @a) (typeRep @b)-  return $ if testUnboxable (typeRep @c) then itransformUnboxed f column else toColumn' (VB.imap f (VB.convert column))-itransform f (GroupedBoxedColumn (column :: VB.Vector (VB.Vector a))) = do-  Refl <- testEquality (typeRep @(VB.Vector a)) (typeRep @b)-  return (toColumn' (VB.imap f column))-itransform f (GroupedUnboxedColumn (column :: VB.Vector (VU.Vector a))) = do-  Refl <- testEquality (typeRep @(VU.Vector a)) (typeRep @b)-  return (toColumn' (VB.imap f column))+itransform+  :: forall b c. (Typeable b, Typeable c, Columnable b, Columnable c)+  => (Int -> b -> c) -> Column -> Maybe Column+itransform f = \case+  BoxedColumn (col :: VB.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just (toColumn' @c (VB.imap f col))+    | otherwise -> Nothing+  UnboxedColumn (col :: VU.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just $+        case sUnbox @c of+          STrue  -> UnboxedColumn (VU.imap f col)+          SFalse -> toColumn' @c (VB.imap f (VB.convert col))+    | otherwise -> Nothing+  OptionalColumn (col :: VB.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just (toColumn' @c (VB.imap f col))+    | otherwise -> Nothing+  GroupedBoxedColumn (col :: VB.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just (toColumn' @c (VB.imap f col))+    | otherwise -> Nothing+  GroupedUnboxedColumn (col :: VB.Vector a)+    | Just Refl <- testEquality (typeRep @a) (typeRep @b)+    -> Just (toColumn' @c (VB.imap f col))+    | otherwise -> Nothing  -- | Filter column with index. ifilterColumn :: forall a . (Columnable a) => (Int -> a -> Bool) -> Column -> Maybe Column@@ -344,23 +401,6 @@   Refl <- testEquality (typeRep @a) (typeRep @b)   return $ GroupedUnboxedColumn $ VG.ifilter f column --- TODO: Expand this to use more predicates.-ifilterColumnF :: Function -> Column -> Maybe Column-ifilterColumnF (ICond (f :: Int -> a -> Bool)) c@(BoxedColumn (column :: VB.Vector b)) = do-  Refl <- testEquality (typeRep @a) (typeRep @b)-  return $ BoxedColumn $ VG.ifilter f column-ifilterColumnF (ICond (f :: Int -> a -> Bool)) c@(UnboxedColumn (column :: VU.Vector b)) = do-  Refl <- testEquality (typeRep @a) (typeRep @b)-  return $ UnboxedColumn $ VG.ifilter f column-ifilterColumnF (ICond (f :: Int -> a -> Bool)) c@(OptionalColumn (column :: VB.Vector b)) = do-  Refl <- testEquality (typeRep @a) (typeRep @b)-  return $ OptionalColumn $ VG.ifilter f column-ifilterColumnF (ICond (f :: Int -> a -> Bool)) c@(GroupedBoxedColumn (column :: VB.Vector b)) = do-  Refl <- testEquality (typeRep @a) (typeRep @b)-  return $ GroupedBoxedColumn $ VG.ifilter f column-ifilterColumnF (ICond (f :: Int -> a -> Bool)) c@(GroupedUnboxedColumn (column :: VB.Vector b)) = do-  Refl <- testEquality (typeRep @a) (typeRep @b)-  return $ GroupedUnboxedColumn $ VG.ifilter f column  ifoldrColumn :: forall a b. (Columnable a, Columnable b) => (Int -> a -> b -> b) -> b -> Column -> Maybe b ifoldrColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = do@@ -429,6 +469,18 @@ zipColumns (UnboxedColumn column) (BoxedColumn other) = BoxedColumn (VB.generate (min (VG.length column) (VG.length other)) (\i -> (column VG.! i, other VG.! i))) zipColumns (UnboxedColumn column) (UnboxedColumn other) = UnboxedColumn (VG.zip column other) {-# INLINE zipColumns #-}++zipWithColumns :: forall a b c . (Columnable a, Columnable b, Columnable c) => (a -> b -> c) -> Column -> Column -> Column+zipWithColumns f (UnboxedColumn (column :: VU.Vector d)) (UnboxedColumn (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of+  Just Refl -> case testEquality (typeRep @b) (typeRep @e) of+    Just Refl -> toColumn' $ VB.zipWith f (VG.convert column) (VG.convert other)+    Nothing -> throw $ TypeMismatchException' (typeRep @b) (show $ typeRep @e) "" "zipWithColumns"+  Nothing -> throw $ TypeMismatchException' (typeRep @a) (show $ typeRep @d) "" "zipWithColumns"+zipWithColumns f left right = let+    left' = toVector @a left+    right' = toVector @b right+  in toColumn' $ VB.zipWith f left' right' +{-# INLINE zipWithColumns #-}  -- Functions for mutable columns (intended for IO). -- Clean this up.

@@ -0,0 +1,135 @@+{-# LANGUAGE ExplicitNamespaces #-}+{-# LANGUAGE GADTs #-}+{-# LANGUAGE InstanceSigs #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RankNTypes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE StrictData #-}+{-# LANGUAGE TypeApplications #-}+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE FlexibleInstances #-}+{-# LANGUAGE UndecidableInstances #-}+{-# LANGUAGE MultiParamTypeClasses #-}+module DataFrame.Internal.Expression where++import qualified Data.Map as M+import Data.Type.Equality (type (:~:)(Refl), TestEquality (testEquality))+import Data.Data (Typeable)+import DataFrame.Internal.Column+import DataFrame.Internal.DataFrame+import DataFrame.Internal.Types+import qualified Data.Text as T+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import Type.Reflection (typeRep)+import DataFrame.Errors (DataFrameException(ColumnNotFoundException))+import Control.Exception (throw)+import Data.Maybe (fromMaybe)++data Expr a where+    Col :: Columnable a => T.Text -> Expr a +    Lit :: Columnable a => a -> Expr a+    Apply :: (Columnable a, Columnable b) => T.Text -> (b -> a) -> Expr b -> Expr a+    BinOp :: (Columnable c, Columnable b, Columnable a) => T.Text -> (c -> b -> a) -> Expr c -> Expr b -> Expr a++interpret :: forall a b . (Columnable a) => DataFrame -> Expr a -> TypedColumn a+interpret df (Lit value) = TColumn $ toColumn' $ V.replicate (fst $ dataframeDimensions df) value+interpret df (Col name) = case getColumn name df of+    Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df)+    Just col -> TColumn col+interpret df (Apply _ (f :: c -> d) value) = let+        (TColumn value') = interpret @c df value+    in TColumn $ fromMaybe (error "transform returned nothing") (transform f value')+interpret df (BinOp _ (f :: c -> d -> e) left right) = let+        (TColumn left') = interpret @c df left+        (TColumn right') = interpret @d df right+    in TColumn $ zipWithColumns f left' right'++instance (Num a, Columnable a) => Num (Expr a) where+    (+) :: Expr a -> Expr a -> Expr a+    (+) = BinOp "add" (+)++    (*) :: Expr a -> Expr a -> Expr a+    (*) = BinOp "mult" (*)++    fromInteger :: Integer -> Expr a+    fromInteger = Lit . fromInteger ++    negate :: Expr a -> Expr a+    negate = Apply "negate" negate++    abs :: Num a => Expr a -> Expr a+    abs = Apply "abs" abs++    signum :: Num a => Expr a -> Expr a+    signum = Apply "signum" signum++instance (Fractional a, Columnable a) => Fractional (Expr a) where+    fromRational :: (Fractional a, Columnable a) => Rational -> Expr a+    fromRational = Lit . fromRational++    (/) :: (Fractional a, Columnable a) => Expr a -> Expr a -> Expr a+    (/) = BinOp "divide" (/)++instance (Floating a, Columnable a) => Floating (Expr a) where+    pi :: (Floating a, Columnable a) => Expr a+    pi = Lit pi+    exp :: (Floating a, Columnable a) => Expr a -> Expr a+    exp = Apply "exp" exp+    log :: (Floating a, Columnable a) => Expr a -> Expr a+    log = Apply "log" log+    sin :: (Floating a, Columnable a) => Expr a -> Expr a+    sin = Apply "sin" sin+    cos :: (Floating a, Columnable a) => Expr a -> Expr a+    cos = Apply "cos" cos+    asin :: (Floating a, Columnable a) => Expr a -> Expr a+    asin = Apply "asin" asin+    acos :: (Floating a, Columnable a) => Expr a -> Expr a+    acos = Apply "acos" acos +    atan :: (Floating a, Columnable a) => Expr a -> Expr a+    atan = Apply "atan" atan+    sinh :: (Floating a, Columnable a) => Expr a -> Expr a+    sinh = Apply "sinh" sinh+    cosh :: (Floating a, Columnable a) => Expr a -> Expr a+    cosh = Apply "cosh" cosh+    asinh :: (Floating a, Columnable a) => Expr a -> Expr a+    asinh = Apply "asinh" sinh+    acosh :: (Floating a, Columnable a) => Expr a -> Expr a+    acosh = Apply "acosh" acosh+    atanh :: (Floating a, Columnable a) => Expr a -> Expr a+    atanh = Apply "atanh" atanh+++instance (Show a) => Show (Expr a) where+    show :: forall a . Show a => Expr a -> String+    show (Col name) = "col@" ++ show (typeRep @a) ++ "(" ++ T.unpack name ++ ")"+    show (Lit value) = show value+    show (Apply name f value) = T.unpack name ++ "(" ++ show value ++ ")"+    show (BinOp name f a b) = T.unpack name ++ "(" ++ show a ++ ", " ++ show b ++ ")" ++col :: Columnable a => T.Text -> Expr a+col = Col++lit :: Columnable a => a -> Expr a+lit = Lit++lift :: (Columnable a, Columnable b) => (a -> b) -> Expr a -> Expr b+lift = Apply "udf"++lift2 :: (Columnable c, Columnable b, Columnable a) => (c -> b -> a) -> Expr c -> Expr b -> Expr a +lift2 = BinOp "udf"++eq :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool+eq = BinOp "eq" (==)++lt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool+lt = BinOp "lt" (<)++gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool+gt = BinOp "gt" (<)++leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool+leq = BinOp "leq" (<=)++geq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool+geq = BinOp "geq" (<=)

@@ -1,83 +0,0 @@-{-# LANGUAGE ExplicitNamespaces #-}-{-# LANGUAGE RankNTypes #-}-{-# LANGUAGE ScopedTypeVariables #-}-{-# LANGUAGE TypeApplications #-}-{-# LANGUAGE ConstraintKinds #-}-{-# LANGUAGE GADTs #-}-{-# LANGUAGE FlexibleInstances #-}-{-# LANGUAGE InstanceSigs #-}-{-# LANGUAGE ViewPatterns #-}-{-# LANGUAGE PatternSynonyms #-}--module DataFrame.Internal.Function where--import qualified Data.Text as T-import qualified Data.Vector as V--import DataFrame.Internal.Types-import Data.Typeable ( Typeable, type (:~:)(Refl) )-import Data.Type.Equality (TestEquality(testEquality))-import Type.Reflection (typeRep, typeOf)---- A GADT to wrap functions so we can have hetegeneous lists of functions.-data Function where-    F1 :: forall a b . (Columnable a, Columnable b) => (a -> b) -> Function-    F2 :: forall a b c . (Columnable a, Columnable b, Columnable c) => (a -> b -> c) -> Function-    F3 :: forall a b c d . (Columnable a, Columnable b, Columnable c, Columnable d) => (a -> b -> c -> d) -> Function-    F4 :: forall a b c d e . (Columnable a, Columnable b, Columnable c, Columnable d, Columnable e) => (a -> b -> c -> d -> e) -> Function-    Cond :: forall a . (Columnable a) => (a -> Bool) -> Function-    ICond :: forall a . (Columnable a) => (Int -> a -> Bool) -> Function---- Helper class to do the actual wrapping-class WrapFunction a where-    wrapFunction :: a -> Function---- Instance for 1-argument functions-instance (Columnable a, Columnable b) => WrapFunction (a -> b) where-    wrapFunction :: (Columnable a, Columnable b) => (a -> b) -> Function-    wrapFunction = F1---- Instance for 2-argument functions-instance {-# INCOHERENT #-} (Columnable a, Columnable b, Columnable c) => WrapFunction (a -> b -> c) where-    wrapFunction :: (Columnable a, Columnable b, Columnable c) => (a -> b -> c) -> Function-    wrapFunction = F2---- Instance for 3-argument functions-instance {-# INCOHERENT #-} (Columnable a, Columnable b, Columnable c, Columnable d) => WrapFunction (a -> b -> c -> d) where-    wrapFunction :: (Columnable a, Columnable b, Columnable c, Columnable d) => (a -> b -> c -> d) -> Function-    wrapFunction = F3--instance {-# INCOHERENT #-} (Columnable a, Columnable b, Columnable c, Columnable d, Columnable e) => WrapFunction (a -> b -> c -> d -> e) where-    wrapFunction :: (Columnable a, Columnable b, Columnable c, Columnable d, Columnable e) => (a -> b -> c -> d -> e) -> Function-    wrapFunction = F4---- The main function that wraps arbitrary functions-func :: forall fn . WrapFunction fn => fn -> Function-func = wrapFunction--pattern Empty :: V.Vector a-pattern Empty <- (V.null -> True) where Empty = V.empty --uncons :: V.Vector a -> Maybe (a, V.Vector a)-uncons Empty = Nothing-uncons v     = Just (V.unsafeHead v, V.unsafeTail v)--pattern (:<|)  :: a -> V.Vector a -> V.Vector a-pattern x :<| xs <- (uncons -> Just (x, xs))--funcApply :: forall c . (Columnable c) => V.Vector RowValue -> Function ->  c-funcApply Empty _ = error "Empty args"-funcApply (Value (x :: a') :<| Empty) (F1 (f :: (a -> b))) = case testEquality (typeRep @a') (typeRep @a) of-        Just Refl -> case testEquality (typeOf (f x)) (typeRep @c) of-            Just Refl -> f x-            Nothing -> error "Result type mismatch"-        Nothing -> error "Arg type mismatch"-funcApply (Value (x :: a') :<| xs) (F2 (f :: (a -> b))) = case testEquality (typeOf x) (typeRep @a) of-        Just Refl -> funcApply xs (F1 (f x))-        Nothing -> error "Arg type mismatch"-funcApply (Value (x :: a') :<| xs) (F3 (f :: (a -> b))) = case testEquality (typeOf x) (typeRep @a) of-        Just Refl -> funcApply xs (F2 (f x))-        Nothing -> error "Arg type mismatch"-funcApply (Value (x :: a') :<| xs) (F4 (f :: (a -> b))) = case testEquality (typeOf x) (typeRep @a) of-        Just Refl -> funcApply xs (F3 (f x))-        Nothing -> error "Arg type mismatch"

@@ -21,10 +21,10 @@  -- We need an "Object" type as an intermediate representation -- for rows. Useful for things like sorting and function application.-type Columnable a = (Typeable a, Show a, Ord a, Eq a)+type Columnable' a = (Typeable a, Show a, Ord a, Eq a)  data RowValue where-    Value :: (Columnable a) => a -> RowValue+    Value :: (Columnable' a) => a -> RowValue  instance Eq RowValue where     (==) :: RowValue -> RowValue -> Bool@@ -42,12 +42,8 @@     show :: RowValue -> String     show (Value a) = show a -toRowValue :: forall a . (Columnable a) => a -> RowValue+toRowValue :: forall a . (Columnable' a) => a -> RowValue toRowValue =  Value---- | Essentially a "functor" instance of our type-erased Column.-class Transformable a where-  transform :: forall b c . (Columnable b, Columnable c) => (b -> c) -> a -> Maybe a  -- Convenience functions for types. unboxableTypes :: TypeRepList '[Int, Int8, Int16, Int32, Int64,

@@ -4,6 +4,7 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}+{-# LANGUAGE FlexibleContexts #-} module DataFrame.Operations.Aggregation where  import qualified Data.Set as S@@ -22,7 +23,7 @@ import Control.Exception (throw) import Control.Monad (foldM_) import Control.Monad.ST (runST)-import DataFrame.Internal.Column (Column(..), toColumn', getIndicesUnboxed, getIndices)+import DataFrame.Internal.Column (Column(..), toColumn', getIndicesUnboxed, getIndices, Columnable) import DataFrame.Internal.DataFrame (DataFrame(..), empty, getColumn) import DataFrame.Internal.Parsing import DataFrame.Internal.Types

@@ -19,10 +19,9 @@  import Control.Exception ( throw ) import DataFrame.Errors-import DataFrame.Internal.Column ( Column(..), toColumn', toColumn, columnLength, columnTypeString, expandColumn )+import DataFrame.Internal.Column ( Column(..), toColumn', toColumn, columnLength, columnTypeString, expandColumn, Columnable) import DataFrame.Internal.DataFrame (DataFrame(..), getColumn, null, empty) import DataFrame.Internal.Parsing (isNullish)-import DataFrame.Internal.Types (Columnable) import Data.Either import Data.Function (on, (&)) import Data.Maybe@@ -43,7 +42,7 @@ -- | /O(n)/ Adds a vector to the dataframe. insertColumn ::   forall a.-  (Columnable a) =>+  Columnable a =>   -- | Column Name   T.Text ->   -- | Vector to add to column

@@ -5,6 +5,7 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE StrictData #-}+{-# LANGUAGE FlexibleContexts #-} module DataFrame.Operations.Statistics where  import qualified Data.List as L@@ -21,7 +22,6 @@ import DataFrame.Errors (DataFrameException(..)) import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame(..), getColumn, empty)-import DataFrame.Internal.Types (Columnable, transform) import DataFrame.Operations.Core import Data.Foldable (asum) import Data.Maybe (isJust, fromMaybe)

@@ -4,6 +4,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE GADTs #-}+{-# LANGUAGE FlexibleContexts #-} module DataFrame.Operations.Subset where  import qualified Data.List as L@@ -19,9 +20,9 @@ import DataFrame.Errors (DataFrameException(..)) import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame(..), getColumn, empty)-import DataFrame.Internal.Function+import DataFrame.Internal.Expression import DataFrame.Internal.Row (mkRowFromArgs)-import DataFrame.Internal.Types (Columnable, RowValue, toRowValue)+import DataFrame.Internal.Types (RowValue, toRowValue) import DataFrame.Operations.Core import DataFrame.Operations.Transformations (apply) import Data.Function ((&))@@ -96,9 +97,10 @@ --   must appear in the same order as they do in the list. -- -- > filterWhere (["x", "y"], func (\x y -> x + y > 5)) df-filterWhere :: ([T.Text], Function) -> DataFrame -> DataFrame-filterWhere (args, f) df = let-    indexes = VG.ifoldl' (\s i row -> if funcApply @Bool row f then S.insert i s else s) S.empty $ V.generate (fst (dimensions df)) (mkRowFromArgs args df)+filterWhere :: Expr Bool -> DataFrame -> DataFrame+filterWhere expr df = let+    (TColumn col) = interpret @Bool df expr+    (Just indexes) = ifoldlColumn (\s i satisfied -> if satisfied then S.insert i s else s) S.empty col     c' = snd $ dataframeDimensions df     pick idxs col = atIndices idxs <$> col   in df {columns = V.map (pick indexes) (columns df), dataframeDimensions = (S.size indexes, c')}

@@ -2,6 +2,7 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}+{-# LANGUAGE FlexibleContexts #-} module DataFrame.Operations.Transformations where  import qualified Data.List as L@@ -13,11 +14,11 @@  import Control.Exception (throw) import DataFrame.Errors (DataFrameException(..))-import DataFrame.Internal.Column (Column(..), columnTypeString, itransform, ifoldrColumn)+import DataFrame.Internal.Column (Column(..), columnTypeString, itransform, ifoldrColumn, TypedColumn (TColumn), Columnable, transform, unwrapTypedColumn) import DataFrame.Internal.DataFrame (DataFrame(..), getColumn)-import DataFrame.Internal.Function (Function(..), funcApply)+import DataFrame.Internal.Expression import DataFrame.Internal.Row (mkRowFromArgs)-import DataFrame.Internal.Types (Columnable, RowValue, toRowValue, transform)+import DataFrame.Internal.Types (RowValue, toRowValue) import DataFrame.Operations.Core import Data.Maybe import Type.Reflection (typeRep, typeOf)@@ -41,41 +42,11 @@  -- | O(k) Apply a function to a combination of columns in a dataframe and -- add the result into `alias` column.-deriveFrom :: ([T.Text], Function) -> T.Text -> DataFrame -> DataFrame-deriveFrom (args, f) name df = case f of-  (F4 (f' :: a -> b -> c -> d -> e)) -> let-      xs = VG.map (\row -> funcApply @e row f) $ V.generate (fst (dimensions df)) (mkRowFromArgs args df)-    in insertColumn name xs df-  (F3 (f' :: a -> b -> c -> d)) -> let-      xs = VG.map (\row -> funcApply @d row f) $ V.generate (fst (dimensions df)) (mkRowFromArgs args df)-    in insertColumn name xs df-  (F2 (f' :: a -> b -> c)) -> let-      xs = VG.map (\row -> funcApply @c row f) $ V.generate (fst (dimensions df)) (mkRowFromArgs args df)-    in insertColumn name xs df-  (F1 (f' :: a -> b)) -> let-      xs = VG.map (\row -> funcApply @b row f) $ V.generate (fst (dimensions df)) (mkRowFromArgs args df)-    in insertColumn name xs df---- | O(k) Apply a function to a given column in a dataframe and--- add the result into alias column.+derive :: forall a . Columnable a => T.Text -> Expr a -> DataFrame -> DataFrame+derive name expr df = let+    value = interpret @a df expr+  in insertColumn' name (Just (unwrapTypedColumn value)) df -derive ::-  forall b c.-  (Columnable b, Columnable c) =>-  -- | New name-  T.Text ->-  -- | function to apply-  (b -> c) ->-  -- | Derivative column name-  T.Text ->-  -- | DataFrame to apply operation to-  DataFrame ->-  DataFrame-derive alias f columnName d = case getColumn columnName d of-  Nothing -> throw $ ColumnNotFoundException columnName "derive" (map fst $ M.toList $ columnIndices d)-  Just column -> case transform f column of-    Nothing  -> throw $ TypeMismatchException (typeOf column) (typeRep @b) columnName "derive"-    Just res -> insertColumn' alias (Just res) d  -- | O(k * n) Apply a function to given column names in a dataframe. applyMany ::

@@ -23,13 +23,13 @@ testData :: D.DataFrame testData = D.fromList values -deriveFromWAI :: Test-deriveFromWAI = TestCase (assertEqual "deriveFrom works when function args align"+deriveWAI :: Test+deriveWAI = TestCase (assertEqual "derive works with column expression"                                 (Just $ DI.BoxedColumn (V.fromList (zipWith (\n c -> show n ++ [c]) [1..26] ['a'..'z'])))-                                (DI.getColumn "test4" $ D.deriveFrom (-                                    ["test1", "test3"],-                                    D.func (\(n :: Int) (c :: Char) -> show n ++ [c])) "test4" testData))+                                (DI.getColumn "test4" $ D.derive "test4" (+                                    D.lift2 (++) (D.lift show (D.col @Int "test1")) (D.lift (: ([] :: [Char])) (D.col @Char "test3"))+                                    ) testData))  tests :: [Test]-tests = [ TestLabel "deriveFromWAI" deriveFromWAI+tests = [ TestLabel "deriveWAI" deriveWAI         ]