packages feed

dataframe-operations 2.6.0.0 → 2.7.0.0

raw patch · 15 files changed

+522/−270 lines, 15 filesdep ~dataframe-core

Dependency ranges changed: dataframe-core

Files

dataframe-operations.cabal view
@@ -1,6 +1,6 @@ cabal-version:      3.4 name:               dataframe-operations-version:            2.6.0.0+version:            2.7.0.0 synopsis:           Column operations, expression DSL, and statistics for the dataframe ecosystem. description:     Untyped column operations (select, filter, sort, join, groupBy,@@ -52,6 +52,7 @@                         DataFrame.Typed.Apply                         DataFrame.Typed.Expr                         DataFrame.Typed.Expr.Extra+                        DataFrame.Typed.Functions                         DataFrame.Typed.Join                         DataFrame.Typed.Operations                         DataFrame.Typed.Sampling@@ -59,7 +60,7 @@     build-depends:      base >= 4 && < 5,                         bytestring >= 0.11 && < 0.14,                         containers >= 0.6.7 && < 0.10,-                        dataframe-core >= 2.6 && < 2.7,+                        dataframe-core >= 2.7 && < 2.8,                         dataframe-parsing >= 2.2 && < 2.3,                         random >= 1.2 && < 2,                         regex-tdfa >= 1.3.0 && < 2,
src-internal/DataFrame/Internal/Statistics.hs view
@@ -14,6 +14,10 @@ import Control.Monad.ST (runST) import DataFrame.Errors (DataFrameException (..)) +-- | The present values of a nullable vector, as doubles, in reverse order.+optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double+optionalToDoubleVector = VU.fromList . V.foldl' (\acc e -> maybe acc ((: acc) . realToFrac) e) []+ mean' :: (Real a, VU.Unbox a) => VU.Vector a -> Double mean' samp     | VU.null samp = throw $ EmptyDataSetException "mean"
src/DataFrame/Expression/Parse.hs view
@@ -61,10 +61,10 @@ schemaOf df = M.fromList [(c, t) | c <- columnNames df, Just t <- [typeOf c]]   where     typeOf c-        | ok (F.col @Int c) = Just TInt-        | ok (F.col @Double c) = Just TDouble-        | ok (F.col @T.Text c) = Just TText-        | ok (F.col @Bool c) = Just TBool+        | ok (F.int c) = Just TInt+        | ok (F.double c) = Just TDouble+        | ok (F.text c) = Just TText+        | ok (F.bool c) = Just TBool         | otherwise = Nothing     ok :: forall a. (Columnable a) => Expr a -> Bool     ok e = either (const False) (const True) (columnAsVector e df)@@ -116,10 +116,10 @@         call name args'   where     colOf c t = case t of-        TInt -> SomeExpr (F.col @Int c)-        TDouble -> SomeExpr (F.col @Double c)-        TText -> SomeExpr (F.col @T.Text c)-        TBool -> SomeExpr (F.col @Bool c)+        TInt -> SomeExpr (F.int c)+        TDouble -> SomeExpr (F.double c)+        TText -> SomeExpr (F.text c)+        TBool -> SomeExpr (F.bool c)  -- | Make two operands the same type; a numeric literal adopts the other's. unify :: SomeExpr -> SomeExpr -> Either String Same
src/DataFrame/Functions.hs view
@@ -27,7 +27,6 @@ import DataFrame.Internal.Statistics  import Control.Applicative-import qualified Data.Char as Char import Data.Either import Data.Function (on) import Data.Int@@ -56,7 +55,8 @@  lift :: (Columnable a, Columnable b) => (a -> b) -> Expr a -> Expr b lift f =-    Unary (MkUnaryOp{unaryFn = f, unaryName = "unaryUdf", unarySymbol = Nothing})+    Unary+        (MkUnaryOp{unaryFn = f, unaryId = Anonymous "unaryUdf", unarySymbol = Nothing})  lift2 ::     (Columnable c, Columnable b, Columnable a) =>@@ -68,7 +68,7 @@     Binary         ( MkBinaryOp             { binaryFn = f-            , binaryName = "binaryUdf"+            , binaryId = Anonymous "binaryUdf"             , binarySymbol = Nothing             , binaryCommutative = False             , binaryPrecedence = 0@@ -83,7 +83,12 @@     Expr (NullLift1Result a r) nullLift f =     Unary-        (MkUnaryOp{unaryFn = applyNull1 f, unaryName = "nullLift", unarySymbol = Nothing})+        ( MkUnaryOp+            { unaryFn = applyNull1 f+            , unaryId = Anonymous "nullLift"+            , unarySymbol = Nothing+            }+        )  -- | Lift a binary function over nullable or non-nullable column expressions. nullLift2 ::@@ -96,7 +101,7 @@     Binary         ( MkBinaryOp             { binaryFn = applyNull2 f-            , binaryName = "nullLift2"+            , binaryId = Anonymous "nullLift2"             , binarySymbol = Nothing             , binaryCommutative = False             , binaryPrecedence = 0@@ -105,24 +110,27 @@  -- | Lenient coercion of a column to @Maybe a@; failures become 'Nothing'. cast :: forall a. (Columnable a, Read a) => T.Text -> Expr (Maybe a)-cast colName = CastWith colName "cast" (either (const Nothing) Just)+cast colName = CastWith colName (Builtin CCast) (either (const Nothing) Just)  -- | Lenient coercion that substitutes a default for unconvertible values. castWithDefault :: forall a. (Columnable a, Read a) => a -> T.Text -> Expr a castWithDefault def colName =-    CastWith colName ("castWithDefault:" <> T.pack (show def)) (fromRight def)+    CastWith+        colName+        (Builtin (CCastWithDefault (T.pack (show def))))+        (fromRight def)  -- | Lenient coercion to @Either T.Text a@, keeping unparsed values as 'Left'. castEither ::     forall a. (Columnable a, Read a) => T.Text -> Expr (Either T.Text a)-castEither colName = CastWith colName "castEither" (either (Left . T.pack) Right)+castEither colName = CastWith colName (Builtin CCastEither) (either (Left . T.pack) Right)  -- | Lenient coercion that crashes at evaluation if a @Nothing@ is found. unsafeCast :: forall a. (Columnable a, Read a) => T.Text -> Expr a unsafeCast colName =     CastWith         colName-        "unsafeCast"+        (Builtin CUnsafeCast)         (fromRight (error "unsafeCast: unexpected Nothing in column"))  castExpr ::@@ -130,13 +138,13 @@     (Columnable b, Columnable src, Read b) =>     Expr src ->     Expr (Maybe b)-castExpr = CastExprWith @b @(Maybe b) @src "castExpr" (either (const Nothing) Just)+castExpr = CastExprWith @b @(Maybe b) @src (Builtin CCast) (either (const Nothing) Just)  castExprWithDefault ::     forall b src. (Columnable b, Columnable src, Read b) => b -> Expr src -> Expr b castExprWithDefault def =     CastExprWith @b @b @src-        ("castExprWithDefault:" <> T.pack (show def))+        (Builtin (CCastWithDefault (T.pack (show def))))         (fromRight def)  castExprEither ::@@ -146,14 +154,14 @@     Expr (Either T.Text b) castExprEither =     CastExprWith @b @(Either T.Text b) @src-        "castExprEither"+        (Builtin CCastEither)         (either (Left . T.pack) Right)  unsafeCastExpr ::     forall b src. (Columnable b, Columnable src, Read b) => Expr src -> Expr b unsafeCastExpr =     CastExprWith @b @b @src-        "unsafeCastExpr"+        (Builtin CUnsafeCast)         (fromRight (error "unsafeCastExpr: unexpected Nothing in column"))  toDouble :: (Columnable a, Real a) => Expr a -> Expr Double@@ -161,34 +169,34 @@     Unary         ( MkUnaryOp             { unaryFn = realToFrac-            , unaryName = "toDouble"+            , unaryId = Builtin UToDouble             , unarySymbol = Nothing             }         )  infix 8 `div` div :: (Integral a, Columnable a) => Expr a -> Expr a -> Expr a-div = lift2Decorated Prelude.div "div" (Just "//") False 7+div = lift2Builtin BDiv Prelude.div (Just "//") False 7  mod :: (Integral a, Columnable a) => Expr a -> Expr a -> Expr a-mod = lift2Decorated Prelude.mod "mod" Nothing False 7+mod = lift2Builtin BMod Prelude.mod Nothing False 7  eq :: (Columnable a, Eq a, a ~ BaseType a) => Expr a -> Expr a -> Expr Bool-eq = lift2Decorated (==) "eq" (Just "==") True 4+eq = lift2Builtin BEq (==) (Just "==") True 4  lt :: (Columnable a, Ord a, a ~ BaseType a) => Expr a -> Expr a -> Expr Bool-lt = lift2Decorated (<) "lt" (Just "<") False 4+lt = lift2Builtin BLt (<) (Just "<") False 4  gt :: (Columnable a, Ord a, a ~ BaseType a) => Expr a -> Expr a -> Expr Bool-gt = lift2Decorated (>) "gt" (Just ">") False 4+gt = lift2Builtin BGt (>) (Just ">") False 4  leq ::     (Columnable a, Ord a, Eq a, a ~ BaseType a) => Expr a -> Expr a -> Expr Bool-leq = lift2Decorated (<=) "leq" (Just "<=") False 4+leq = lift2Builtin BLeq (<=) (Just "<=") False 4  geq ::     (Columnable a, Ord a, Eq a, a ~ BaseType a) => Expr a -> Expr a -> Expr Bool-geq = lift2Decorated (>=) "geq" (Just ">=") False 4+geq = lift2Builtin BGeq (>=) (Just ">=") False 4  and :: Expr Bool -> Expr Bool -> Expr Bool and = (.&&)@@ -199,10 +207,10 @@ not :: Expr Bool -> Expr Bool not =     Unary-        (MkUnaryOp{unaryFn = Prelude.not, unaryName = "not", unarySymbol = Just "~"})+        (MkUnaryOp{unaryFn = Prelude.not, unaryId = Builtin UNot, unarySymbol = Just "~"})  count :: (Columnable a) => Expr a -> Expr Int-count = Agg (MergeAgg "count" (0 :: Int) (\c _ -> c + 1) (+) id)+count = Agg (MergeAgg (Builtin ACount) (0 :: Int) (\c _ -> c + 1) (+) id) {-# SPECIALIZE count :: Expr Double -> Expr Int #-} {-# SPECIALIZE count :: Expr Float -> Expr Int #-} {-# SPECIALIZE count :: Expr Int -> Expr Int #-}@@ -217,7 +225,7 @@ countDistinct =     Agg         ( MergeAgg-            "countDistinct"+            (Builtin ACountDistinct)             (S.empty :: S.Set a)             (flip S.insert)             S.union@@ -237,7 +245,7 @@ {-# INLINE countAll #-}  collect :: (Columnable a) => Expr a -> Expr [a]-collect = Agg (FoldAgg "collect" (Just []) (flip (:)))+collect = Agg (FoldAgg (Builtin ACollect) (Just []) (flip (:))) {-# SPECIALIZE collect :: Expr Double -> Expr [Double] #-} {-# SPECIALIZE collect :: Expr Float -> Expr [Float] #-} {-# SPECIALIZE collect :: Expr Int -> Expr [Int] #-}@@ -247,7 +255,7 @@ mode =     Agg         ( CollectAgg-            "mode"+            (Builtin AMode)             ( fst                 . L.maximumBy (compare `on` snd)                 . M.toList@@ -264,7 +272,7 @@ {-# INLINEABLE mode #-}  minimum :: (Columnable a, Ord a) => Expr a -> Expr a-minimum = Agg (FoldAgg "minimum" Nothing Prelude.min)+minimum = Agg (FoldAgg (Builtin AMinimum) Nothing Prelude.min) {-# SPECIALIZE DataFrame.Functions.minimum :: Expr Double -> Expr Double #-} {-# SPECIALIZE DataFrame.Functions.minimum :: Expr Float -> Expr Float #-} {-# SPECIALIZE DataFrame.Functions.minimum :: Expr Int -> Expr Int #-}@@ -275,7 +283,7 @@ {-# INLINEABLE DataFrame.Functions.minimum #-}  maximum :: (Columnable a, Ord a) => Expr a -> Expr a-maximum = Agg (FoldAgg "maximum" Nothing Prelude.max)+maximum = Agg (FoldAgg (Builtin AMaximum) Nothing Prelude.max) {-# SPECIALIZE DataFrame.Functions.maximum :: Expr Double -> Expr Double #-} {-# SPECIALIZE DataFrame.Functions.maximum :: Expr Float -> Expr Float #-} {-# SPECIALIZE DataFrame.Functions.maximum :: Expr Int -> Expr Int #-}@@ -286,7 +294,7 @@ {-# INLINEABLE DataFrame.Functions.maximum #-}  sum :: forall a. (Columnable a, Num a) => Expr a -> Expr a-sum = Agg (FoldAgg "sum" Nothing (+))+sum = Agg (FoldAgg (Builtin ASum) Nothing (+)) {-# SPECIALIZE DataFrame.Functions.sum :: Expr Double -> Expr Double #-} {-# SPECIALIZE DataFrame.Functions.sum :: Expr Float -> Expr Float #-} {-# SPECIALIZE DataFrame.Functions.sum :: Expr Int -> Expr Int #-}@@ -297,7 +305,7 @@ {-# INLINEABLE DataFrame.Functions.sum #-}  sumMaybe :: forall a. (Columnable a, Num a) => Expr (Maybe a) -> Expr a-sumMaybe = Agg (CollectAgg "sumMaybe" (P.sum . Maybe.catMaybes . V.toList))+sumMaybe = Agg (CollectAgg (Builtin ASumMaybe) (P.sum . Maybe.catMaybes . V.toList)) {-# SPECIALIZE sumMaybe :: Expr (Maybe Double) -> Expr Double #-} {-# SPECIALIZE sumMaybe :: Expr (Maybe Float) -> Expr Float #-} {-# SPECIALIZE sumMaybe :: Expr (Maybe Int) -> Expr Int #-}@@ -311,7 +319,7 @@ mean =     Agg         ( MergeAgg-            "mean"+            (Builtin AMean)             ((0.0, 0) :: (Double, Int))             (\(!s, !c) x -> (s + realToFrac x, c + 1))             (\(!s1, !c1) (!s2, !c2) -> (s1 + s2, c1 + c2))@@ -328,29 +336,28 @@  -- | The k largest values per group. topK :: (Columnable a, Ord a) => Int -> Expr a -> Expr [a]-topK = kExtremes "topK" (>)+topK = kExtremes True {-# INLINEABLE topK #-}  -- | The k smallest values per group. bottomK :: (Columnable a, Ord a) => Int -> Expr a -> Expr [a]-bottomK = kExtremes "bottomK" (<)+bottomK = kExtremes False {-# INLINEABLE bottomK #-} --- | Top k values after applying an ordering function, dropping NaNs.+-- | The k largest (True) or smallest (False) values, dropping NaNs. kExtremes ::-    forall a.-    (Columnable a, Ord a) =>-    T.Text -> (a -> a -> Bool) -> Int -> Expr a -> Expr [a]-kExtremes opName wins k =+    forall a. (Columnable a, Ord a) => Bool -> Int -> Expr a -> Expr [a]+kExtremes largest k =     Agg         ( MergeAgg-            (opName <> "_" <> T.pack (show k))+            (Builtin (if largest then ATopK k else ABottomK k))             []             step             (L.foldl' step)             id         )   where+    wins = if largest then (>) else (<)     step acc x         | isNaNLike x = acc         | otherwise = insert x acc@@ -366,7 +373,7 @@         | otherwise = False  meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double-meanMaybe = Agg (CollectAgg "meanMaybe" (mean' . optionalToDoubleVector))+meanMaybe = Agg (CollectAgg (Builtin AMeanMaybe) (mean' . optionalToDoubleVector)) {-# SPECIALIZE meanMaybe :: Expr (Maybe Double) -> Expr Double #-} {-# SPECIALIZE meanMaybe :: Expr (Maybe Float) -> Expr Double #-} {-# SPECIALIZE meanMaybe :: Expr (Maybe Int) -> Expr Double #-}@@ -377,7 +384,7 @@ {-# INLINEABLE meanMaybe #-}  variance :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double-variance = Agg (CollectAgg "variance" variance')+variance = Agg (CollectAgg (Builtin AVariance) variance') {-# SPECIALIZE variance :: Expr Double -> Expr Double #-} {-# SPECIALIZE variance :: Expr Float -> Expr Double #-} {-# SPECIALIZE variance :: Expr Int -> Expr Double #-}@@ -388,7 +395,7 @@ {-# INLINEABLE variance #-}  median :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double-median = Agg (CollectAgg "median" median')+median = Agg (CollectAgg (Builtin AMedian) median') {-# SPECIALIZE median :: Expr Double -> Expr Double #-} {-# SPECIALIZE median :: Expr Float -> Expr Double #-} {-# SPECIALIZE median :: Expr Int -> Expr Double #-}@@ -399,7 +406,7 @@ {-# INLINEABLE median #-}  medianMaybe :: (Columnable a, Real a) => Expr (Maybe a) -> Expr Double-medianMaybe = Agg (CollectAgg "meanMaybe" (median' . optionalToDoubleVector))+medianMaybe = Agg (CollectAgg (Builtin AMedianMaybe) (median' . optionalToDoubleVector)) {-# SPECIALIZE medianMaybe :: Expr (Maybe Double) -> Expr Double #-} {-# SPECIALIZE medianMaybe :: Expr (Maybe Float) -> Expr Double #-} {-# SPECIALIZE medianMaybe :: Expr (Maybe Int) -> Expr Double #-}@@ -409,23 +416,16 @@ {-# SPECIALIZE medianMaybe :: Expr (Maybe Int64) -> Expr Double #-} {-# INLINEABLE medianMaybe #-} -optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double-optionalToDoubleVector =-    VU.fromList-        . V.foldl'-            (\acc e -> if Maybe.isJust e then realToFrac (Maybe.fromMaybe 0 e) : acc else acc)-            []- percentile :: Int -> Expr Double -> Expr Double percentile n =     Agg         ( CollectAgg-            (T.pack $ "percentile " ++ show n)+            (Builtin (APercentile n))             (percentile' n)         )  stddev :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double-stddev = Agg (CollectAgg "stddev" (sqrt . variance'))+stddev = Agg (CollectAgg (Builtin AStddev) (sqrt . variance')) {-# SPECIALIZE stddev :: Expr Double -> Expr Double #-} {-# SPECIALIZE stddev :: Expr Float -> Expr Double #-} {-# SPECIALIZE stddev :: Expr Int -> Expr Double #-}@@ -436,7 +436,9 @@ {-# INLINEABLE stddev #-}  stddevMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double-stddevMaybe = Agg (CollectAgg "stddevMaybe" (sqrt . variance' . optionalToDoubleVector))+stddevMaybe =+    Agg+        (CollectAgg (Builtin AStddevMaybe) (sqrt . variance' . optionalToDoubleVector)) {-# SPECIALIZE stddevMaybe :: Expr (Maybe Double) -> Expr Double #-} {-# SPECIALIZE stddevMaybe :: Expr (Maybe Float) -> Expr Double #-} {-# SPECIALIZE stddevMaybe :: Expr (Maybe Int) -> Expr Double #-}@@ -450,21 +452,21 @@ zScore c = (c - mean c) / stddev c  pow :: (Columnable a, Num a) => Expr a -> Int -> Expr a-pow expr i = lift2Decorated (^) "pow" (Just "^") False 8 expr (Lit i)+pow expr i = lift2Builtin BPow (^) (Just "^") False 8 expr (Lit i) {-# SPECIALIZE pow :: Expr Double -> Int -> Expr Double #-} {-# SPECIALIZE pow :: Expr Float -> Int -> Expr Float #-} {-# SPECIALIZE pow :: Expr Int -> Int -> Expr Int #-} {-# INLINEABLE pow #-}  relu :: (Columnable a, Num a, Ord a) => Expr a -> Expr a-relu = liftDecorated (Prelude.max 0) "relu" Nothing+relu = liftBuiltin URelu (Prelude.max 0) Nothing {-# SPECIALIZE relu :: Expr Double -> Expr Double #-} {-# SPECIALIZE relu :: Expr Float -> Expr Float #-} {-# SPECIALIZE relu :: Expr Int -> Expr Int #-} {-# INLINEABLE relu #-}  min :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr a-min = lift2Decorated Prelude.min "min" Nothing True 1+min = lift2Builtin BMin Prelude.min Nothing True 1 {-# SPECIALIZE DataFrame.Functions.min ::     Expr Double -> Expr Double -> Expr Double     #-}@@ -475,7 +477,7 @@ {-# INLINEABLE DataFrame.Functions.min #-}  max :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr a-max = lift2Decorated Prelude.max "max" Nothing True 1+max = lift2Builtin BMax Prelude.max Nothing True 1 {-# SPECIALIZE DataFrame.Functions.max ::     Expr Double -> Expr Double -> Expr Double     #-}@@ -492,31 +494,31 @@     a ->     (a -> b -> a) ->     Expr a-reduce expr start f = Agg (FoldAgg "foldUdf" (Just start) f) expr+reduce expr start f = Agg (FoldAgg (Anonymous "foldUdf") (Just start) f) expr {-# INLINEABLE reduce #-}  toMaybe :: (Columnable a) => Expr a -> Expr (Maybe a)-toMaybe = liftDecorated Just "toMaybe" Nothing+toMaybe = liftBuiltin UToMaybe Just Nothing {-# SPECIALIZE toMaybe :: Expr Double -> Expr (Maybe Double) #-} {-# SPECIALIZE toMaybe :: Expr Float -> Expr (Maybe Float) #-} {-# SPECIALIZE toMaybe :: Expr Int -> Expr (Maybe Int) #-} {-# INLINEABLE toMaybe #-}  fromMaybe :: (Columnable a) => a -> Expr (Maybe a) -> Expr a-fromMaybe d = liftDecorated (Maybe.fromMaybe d) "fromMaybe" Nothing+fromMaybe d = liftBuiltin (UFromMaybe (T.pack (show d))) (Maybe.fromMaybe d) Nothing {-# SPECIALIZE fromMaybe :: Double -> Expr (Maybe Double) -> Expr Double #-} {-# SPECIALIZE fromMaybe :: Float -> Expr (Maybe Float) -> Expr Float #-} {-# SPECIALIZE fromMaybe :: Int -> Expr (Maybe Int) -> Expr Int #-} {-# INLINEABLE fromMaybe #-}  isJust :: (Columnable a) => Expr (Maybe a) -> Expr Bool-isJust = liftDecorated Maybe.isJust "isJust" Nothing+isJust = liftBuiltin UIsJust Maybe.isJust Nothing {-# SPECIALIZE isJust :: Expr (Maybe Double) -> Expr Bool #-} {-# SPECIALIZE isJust :: Expr (Maybe Int) -> Expr Bool #-} {-# INLINEABLE isJust #-}  isNothing :: (Columnable a) => Expr (Maybe a) -> Expr Bool-isNothing = liftDecorated Maybe.isNothing "isNothing" Nothing+isNothing = liftBuiltin UIsNothing Maybe.isNothing Nothing {-# SPECIALIZE isNothing :: Expr (Maybe Double) -> Expr Bool #-} {-# SPECIALIZE isNothing :: Expr (Maybe Int) -> Expr Bool #-} {-# INLINEABLE isNothing #-}@@ -532,7 +534,7 @@ {-# INLINEABLE isNotNull #-}  fromJust :: (Columnable a) => Expr (Maybe a) -> Expr a-fromJust = liftDecorated Maybe.fromJust "fromJust" Nothing+fromJust = liftBuiltin UFromJust Maybe.fromJust Nothing {-# SPECIALIZE fromJust :: Expr (Maybe Double) -> Expr Double #-} {-# SPECIALIZE fromJust :: Expr (Maybe Int) -> Expr Int #-} {-# INLINEABLE fromJust #-}@@ -543,7 +545,7 @@     (a -> b) ->     Expr (Maybe a) ->     Expr (Maybe b)-whenPresent f = liftDecorated (fmap f) "whenPresent" Nothing+whenPresent f = Unary (MkUnaryOp (fmap f) (Anonymous "whenPresent") Nothing) {-# INLINEABLE whenPresent #-}  whenBothPresent ::@@ -553,7 +555,9 @@     Expr (Maybe a) ->     Expr (Maybe b) ->     Expr (Maybe c)-whenBothPresent f = lift2Decorated (\l r -> f <$> l <*> r) "whenBothPresent" Nothing False 0+whenBothPresent f =+    Binary+        (MkBinaryOp (\l r -> f <$> l <*> r) (Anonymous "whenBothPresent") Nothing False 0) {-# INLINEABLE whenBothPresent #-}  recode ::@@ -566,7 +570,7 @@     Unary         ( MkUnaryOp             { unaryFn = (`lookup` mapping)-            , unaryName = "recode " <> T.pack (show mapping)+            , unaryId = Builtin (URecode (T.pack (show mapping)))             , unarySymbol = Nothing             }         )@@ -592,57 +596,57 @@     Unary         ( MkUnaryOp             { unaryFn = Maybe.fromMaybe d . (`lookup` mapping)-            , unaryName =-                "recodeWithDefault " <> T.pack (show d) <> " " <> T.pack (show mapping)+            , unaryId =+                Builtin (URecodeWithDefault (T.pack (show d)) (T.pack (show mapping)))             , unarySymbol = Nothing             }         )  firstOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)-firstOrNothing = liftDecorated Maybe.listToMaybe "firstOrNothing" Nothing+firstOrNothing = liftBuiltin UFirstOrNothing Maybe.listToMaybe Nothing  lastOrNothing :: (Columnable a) => Expr [a] -> Expr (Maybe a)-lastOrNothing = liftDecorated (Maybe.listToMaybe . reverse) "lastOrNothing" Nothing+lastOrNothing = liftBuiltin ULastOrNothing (Maybe.listToMaybe . reverse) Nothing  -- | Whether the text contains the literal substring. contains :: T.Text -> Expr T.Text -> Expr Bool contains needle expr =-    lift2Decorated (flip T.isInfixOf) "contains" Nothing False 4 expr (Lit needle)+    lift2Builtin BContains (flip T.isInfixOf) Nothing False 4 expr (Lit needle)  -- | Character count. length :: Expr T.Text -> Expr Int-length = liftDecorated T.length "textLength" Nothing+length = liftBuiltin UTextLength T.length Nothing  splitOn :: T.Text -> Expr T.Text -> Expr [T.Text]-splitOn delim = liftDecorated (T.splitOn delim) "splitOn" Nothing+splitOn delim = liftBuiltin (USplitOn delim) (T.splitOn delim) Nothing  match :: T.Text -> Expr T.Text -> Expr (Maybe T.Text) match regex =-    liftDecorated+    liftBuiltin+        (UMatch regex)         ((\r -> if T.null r then Nothing else Just r) . (=~ regex))-        ("match " <> T.pack (show regex))         Nothing  matchAll :: T.Text -> Expr T.Text -> Expr [T.Text] matchAll regex =-    liftDecorated+    liftBuiltin+        (UMatchAll regex)         (getAllTextMatches . (=~ regex))-        ("matchAll " <> T.pack (show regex))         Nothing  parseDate ::     (ParseTime t, Columnable t) => T.Text -> Expr T.Text -> Expr (Maybe t) parseDate format =-    liftDecorated+    liftBuiltin+        (UParseDate format)         (parseTimeM True defaultTimeLocale (T.unpack format) . T.unpack)-        ("parseDate " <> format)         Nothing  daysBetween :: Expr Day -> Expr Day -> Expr Int daysBetween =-    lift2Decorated+    lift2Builtin+        BDaysBetween         (\d1 d2 -> fromIntegral (diffDays d1 d2))-        "daysBetween"         Nothing         False         2@@ -653,71 +657,8 @@     (a -> m b) ->     Expr (m a) ->     Expr (m b)-bind f = liftDecorated (>>= f) "bind" Nothing+bind f = Unary (MkUnaryOp (>>= f) (Anonymous "bind") Nothing)  -- | Window function: evaluate an expression partitioned by the given columns. over :: (Columnable a) => [T.Text] -> Expr a -> Expr a over = Over---- Haskell Report 2.4: https://www.haskell.org/definition/haskell2010.pdf-isReservedId :: T.Text -> Bool-isReservedId t = case t of-    "case" -> True-    "class" -> True-    "data" -> True-    "default" -> True-    "deriving" -> True-    "do" -> True-    "else" -> True-    "foreign" -> True-    "if" -> True-    "import" -> True-    "in" -> True-    "infix" -> True-    "infixl" -> True-    "infixr" -> True-    "instance" -> True-    "let" -> True-    "module" -> True-    "newtype" -> True-    "of" -> True-    "then" -> True-    "type" -> True-    "where" -> True-    _ -> False--isVarId :: T.Text -> Bool-isVarId t = case T.uncons t of-    -- '_' is skipped on purpose so "Name" and "_Name_" cannot collide.-    Just (c, _) -> Char.isLower c && Char.isAlpha c-    Nothing -> False--isHaskellIdentifier :: T.Text -> Bool-isHaskellIdentifier t = Prelude.not (isVarId t) || isReservedId t--sanitize :: T.Text -> T.Text-sanitize t-    | isValid = t-    | isHaskellIdentifier t' = "_" <> t' <> "_"-    | otherwise = t'-  where-    isValid =-        Prelude.not (isHaskellIdentifier t)-            && isVarId t-            && T.all Char.isAlphaNum t-    t' = T.map replaceInvalidCharacters . T.filter (Prelude.not . parentheses) $ t-    replaceInvalidCharacters c-        | Char.isUpper c = Char.toLower c-        | Char.isSpace c = '_'-        | Char.isPunctuation c = '_' -- '-' will also become a '_'-        | Char.isSymbol c = '_'-        | Char.isAlphaNum c = c -- Blanket condition-        | otherwise = '_' -- If we're unsure we'll default to an underscore-    parentheses c = case c of-        '(' -> True-        ')' -> True-        '{' -> True-        '}' -> True-        '[' -> True-        ']' -> True-        _ -> False
src/DataFrame/Monad.hs view
@@ -20,7 +20,7 @@     deriveM,     insertM,     renameM,-    filterWhereM,+    filterM,     sampleM,     takeM,     dropM,@@ -103,8 +103,8 @@ renameM (Col oldName) newName = execWithExpr newName (D.rename oldName newName) renameM expr newName = deriveM newName expr -filterWhereM :: Expr Bool -> FrameM ()-filterWhereM p = modifyM (D.filterWhere p)+filterM :: Expr Bool -> FrameM ()+filterM p = modifyM (D.filter p)  sampleM :: (RandomGen g) => g -> Double -> FrameM () sampleM pureGen p = modifyM (D.sample pureGen p)
src/DataFrame/Operations/Join.hs view
@@ -14,6 +14,7 @@      -- * Joins     join,+    joinOn,     innerJoin,     leftJoin,     rightJoin,@@ -98,6 +99,20 @@ join LEFT xs right = leftJoin xs right join RIGHT xs right = rightJoin xs right join FULL_OUTER xs right = fullOuterJoin xs right++-- | Join on @(leftKey, rightKey)@ pairs; the result keeps the left names.+joinOn ::+    JoinType ->+    [(T.Text, T.Text)] ->+    DataFrame ->+    DataFrame ->+    DataFrame+joinOn jt keyPairs left right =+    join+        jt+        (map fst keyPairs)+        left+        (renameMany [(r, l) | (l, r) <- keyPairs, l /= r] right)  -- | Build-side row count above which the hash join switches strategy. joinStrategyThreshold :: Int
src/DataFrame/Operations/Permutation.hs view
@@ -9,6 +9,7 @@ module DataFrame.Operations.Permutation (     -- * Sorting     SortOrder (..),+    sortOrderColumns,     sortBy,     topBy, @@ -75,6 +76,12 @@     (==) (Desc _) (Desc _) = True     (==) _ _ = False +instance Show SortOrder where+    showsPrec d (Asc e) = showParen (d > 10) (showString "Asc " . showsPrec 11 e)+    showsPrec d (Desc e) =+        showParen (d > 10) (showString "Desc " . showsPrec 11 e)++-- | Columns a sort order reads. sortOrderColumns :: SortOrder -> [T.Text] sortOrderColumns (Asc e) = getColumns e sortOrderColumns (Desc e) = getColumns e
src/DataFrame/Operations/Statistics.hs view
@@ -235,13 +235,6 @@     _ -> Nothing {-# INLINE _getColumnAsDouble #-} -optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double-optionalToDoubleVector =-    VU.fromList-        . V.foldl'-            (\acc e -> if isJust e then realToFrac (fromMaybe 0 e) : acc else acc)-            []- -- | Calculates the sum of a given column as a standalone value. sum ::     forall a. (Columnable a, Num a) => Expr a -> DataFrame -> a
src/DataFrame/Operations/Subset.hs view
@@ -23,7 +23,6 @@     -- * Filtering     filter,     filterBy,-    filterWhere,     filterJust,     filterNothing,     filterAllJust,@@ -190,25 +189,28 @@ clip :: Int -> Int -> Int -> Int clip n left right = min right $ max n left --- | O(n * k) Filter rows by a given condition.-filter ::+-- | O(n * k) Keep rows whose value of the expression satisfies the predicate.+filterBy ::     forall a.     (Columnable a) =>-    -- | Column to filter by-    Expr a ->     -- | Filter condition     (a -> Bool) ->+    -- | Column to filter by+    Expr a ->     -- | Dataframe to filter     DataFrame ->     DataFrame-filter e@(Col filterColumnName) condition df = case getColumn filterColumnName df of+filterBy condition e@(Col filterColumnName) df = case getColumn filterColumnName df of     Nothing ->         throw $-            ColumnsNotFoundException [filterColumnName] "filter" (M.keys $ columnIndices df)+            ColumnsNotFoundException+                [filterColumnName]+                "filterBy"+                (M.keys $ columnIndices df)     Just c@(PackedText _ _) ->-        filter e condition (insertColumn filterColumnName (materializePacked c) df)+        filterBy condition e (insertColumn filterColumnName (materializePacked c) df)     Just c@(MergedColumn _ _) ->-        filter e condition (insertColumn filterColumnName (materializeMerged c) df)+        filterBy condition e (insertColumn filterColumnName (materializeMerged c) df)     Just _col@(BoxedColumn bm (column :: V.Vector b)) ->         case testEquality (typeRep @a) (typeRep @b) of             Just Refl -> filterByVector filterColumnName column condition df@@ -234,7 +236,7 @@                         Nothing -> filterByVector filterColumnName column condition df                     Nothing -> filterByVector filterColumnName column condition df                 _ -> filterByVector filterColumnName column condition df-filter expr condition df =+filterBy condition expr df =     let         (TColumn col') = case interpret @a df (normalize expr) of             Left e -> throw e@@ -261,7 +263,7 @@                     { userType = Right $ typeRep @a                     , expectedType = Right $ typeRep @b                     , errorColumnName = Just (T.unpack filterColumnName)-                    , callingFunctionName = Just "filter"+                    , callingFunctionName = Just "filterBy"                     }                 )     Just Refl ->@@ -273,13 +275,9 @@                 , dataframeDimensions = (VG.length ixs, snd (dataframeDimensions df))                 } --- | O(k) A version of filter where the predicate comes first.-filterBy :: (Columnable a) => (a -> Bool) -> Expr a -> DataFrame -> DataFrame-filterBy = flip filter- -- | O(k) Filter the dataframe with a boolean expression.-filterWhere :: Expr Bool -> DataFrame -> DataFrame-filterWhere expr df =+filter :: Expr Bool -> DataFrame -> DataFrame+filter expr df =     let         (TColumn col') = case interpret @Bool df (normalize expr) of             Left e -> throw e@@ -305,9 +303,9 @@             ColumnsNotFoundException [colName] "filterJust" (M.keys $ columnIndices df)     Just column | hasMissing column -> case column of         BoxedColumn (Just _) (_col :: V.Vector a) ->-            filter (Col @(Maybe a) colName) isJust df & apply @(Maybe a) fromJust colName+            filterBy isJust (Col @(Maybe a) colName) df & apply @(Maybe a) fromJust colName         UnboxedColumn (Just _) (_col :: VU.Vector a) ->-            filter (Col @(Maybe a) colName) isJust df & apply @(Maybe a) fromJust colName+            filterBy isJust (Col @(Maybe a) colName) df & apply @(Maybe a) fromJust colName         c@(PackedText (Just _) _) ->             filterJust colName (insertColumn colName (materializePacked c) df)         _ -> df@@ -320,8 +318,8 @@         throw $             ColumnsNotFoundException [colName] "filterNothing" (M.keys $ columnIndices df)     Just column | hasMissing column -> case column of-        BoxedColumn (Just _) (_col :: V.Vector a) -> filter (Col @(Maybe a) colName) isNothing df-        UnboxedColumn (Just _) (_col :: VU.Vector a) -> filter (Col @(Maybe a) colName) isNothing df+        BoxedColumn (Just _) (_col :: V.Vector a) -> filterBy isNothing (Col @(Maybe a) colName) df+        UnboxedColumn (Just _) (_col :: VU.Vector a) -> filterBy isNothing (Col @(Maybe a) colName) df         c@(PackedText (Just _) _) ->             filterNothing colName (insertColumn colName (materializePacked c) df)         _ -> df@@ -435,7 +433,7 @@      in         df             & insertColumn (name cRand) rand-            & filterWhere (cRand .>=. Lit (1 - p))+            & filter (cRand .>=. Lit (1 - p))             & exclude [name cRand]  -- | Split a dataset in two: a sample of fraction p and the remainder.@@ -448,10 +446,10 @@         withRand = df & insertColumn (name cRand) rand      in         ( withRand-            & filterWhere (cRand .<=. Lit p)+            & filter (cRand .<=. Lit p)             & exclude [name cRand]         , withRand-            & filterWhere+            & filter                 (cRand .>. Lit p)             & exclude [name cRand]         )@@ -465,12 +463,12 @@         withRand = df & insertColumn (name cRand) rand         partitionSize = 1 / fromIntegral folds         singleFold n d =-            d & filterWhere (cRand .>=. Lit (fromIntegral n * partitionSize))+            d & filter (cRand .>=. Lit (fromIntegral n * partitionSize))         go (-1) _ = []         go n d =             let                 d' = singleFold n d-                d'' = d & filterWhere (cRand .<. Lit (fromIntegral n * partitionSize))+                d'' = d & filter (cRand .<. Lit (fromIntegral n * partitionSize))              in                 d' : go (n - 1) d''      in
src/DataFrame/Typed/Apply.hs view
@@ -8,7 +8,9 @@  -- | Typed column transformations that track schema changes in the types. module DataFrame.Typed.Apply (-    applyColumn,+    apply,+    applyInt,+    applyDouble,     applyMany,     applyWhere,     applyAtIndex,@@ -47,7 +49,7 @@ import DataFrame.Typed.Types (TExpr (..), TypedDataFrame (..))  -- | Map a function over a column, rewriting its element type from @a@ to @b@.-applyColumn ::+apply ::     forall name a b cols.     ( KnownSymbol name     , a ~ SafeLookup name cols@@ -58,11 +60,37 @@     (a -> b) ->     TypedDataFrame cols ->     TypedDataFrame (SetColumnType name b cols)-applyColumn f (TDF df) = unsafeFreeze (D.apply f colName df)+apply f (TDF df) = unsafeFreeze (D.apply f colName df)   where     colName = T.pack (symbolVal (Proxy @name)) --- | Like 'applyColumn' but returns the error instead of throwing.+-- | 'apply' specialised to an @Int@ column.+applyInt ::+    forall name b cols.+    ( KnownSymbol name+    , Int ~ SafeLookup name cols+    , Columnable b+    , AssertPresent name cols+    ) =>+    (Int -> b) ->+    TypedDataFrame cols ->+    TypedDataFrame (SetColumnType name b cols)+applyInt = apply @name++-- | 'apply' specialised to a @Double@ column.+applyDouble ::+    forall name b cols.+    ( KnownSymbol name+    , Double ~ SafeLookup name cols+    , Columnable b+    , AssertPresent name cols+    ) =>+    (Double -> b) ->+    TypedDataFrame cols ->+    TypedDataFrame (SetColumnType name b cols)+applyDouble = apply @name++-- | Like 'apply' but returns the error instead of throwing. safeApply ::     forall name a b cols.     ( KnownSymbol name
src/DataFrame/Typed/Expr.hs view
@@ -105,7 +105,6 @@     module DataFrame.Typed.Expr.Extra, ) where -import Data.Either (fromRight) import Data.Proxy (Proxy (..)) import Data.String (IsString (..)) import qualified Data.Text as T@@ -116,15 +115,16 @@ import DataFrame.Internal.Column (Columnable) import DataFrame.Internal.Column.Types (Promote, PromoteDiv) import DataFrame.Internal.Expression (+    BinOpId (..),     BinUDF (..),     Expr (..),-    UnUDF (..),+    OpRef (..),  ) import DataFrame.Internal.Expression.Operators.Nullable (     BaseType,     DivWidenOp,     NullCmpResult,-    NullLift1Op (applyNull1),+    NullLift1Op,     NullLift1Result,     NullLift2Op (applyNull2),     NullLift2Result,@@ -229,13 +229,13 @@ -- | Lift a unary function into a typed expression. lift ::     (Columnable a, Columnable b) => (a -> b) -> TExpr cols a -> TExpr cols b-lift f (TExpr e) = TExpr (Unary (MkUnaryOp f "unaryUdf" Nothing) e)+lift f (TExpr e) = TExpr (F.lift f e)  -- | Lift a binary function into typed expressions. lift2 ::     (Columnable a, Columnable b, Columnable c) =>     (a -> b -> c) -> TExpr cols a -> TExpr cols b -> TExpr cols c-lift2 f (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp f "binaryUdf" Nothing False 0) a b)+lift2 f (TExpr a) (TExpr b) = TExpr (F.lift2 f a b)  -- | Typed 'nullLift': lift a unary function with nullable propagation. nullLift ::@@ -243,7 +243,7 @@     (BaseType a -> r) ->     TExpr cols a ->     TExpr cols (NullLift1Result a r)-nullLift f (TExpr e) = TExpr (Unary (MkUnaryOp (applyNull1 f) "nullLift" Nothing) e)+nullLift f (TExpr e) = TExpr (F.nullLift f e)  -- | Typed 'nullLift2': lift a binary function with nullable propagation. nullLift2 ::@@ -252,7 +252,7 @@     TExpr cols a ->     TExpr cols b ->     TExpr cols (NullLift2Result a b r)-nullLift2 f (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (applyNull2 f) "nullLift2" Nothing False 0) a b)+nullLift2 f (TExpr a) (TExpr b) = TExpr (F.nullLift2 f a b)  infixl 4 .==., ./=., .<., .<=., .>=., .>. infix 4 .==, ./=, .<, .<=, .>=, .>@@ -264,27 +264,27 @@  (.==.) ::     (Columnable a, Eq a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool-(.==.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (==) "eq" (Just "==") True 4) a b)+(.==.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (==) (Builtin BEq) (Just "==") True 4) a b)  (./=.) ::     (Columnable a, Eq a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool-(./=.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (/=) "neq" (Just "/=") True 4) a b)+(./=.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (/=) (Builtin BNeq) (Just "/=") True 4) a b)  (.<.) ::     (Columnable a, Ord a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool-(.<.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (<) "lt" (Just "<") False 4) a b)+(.<.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (<) (Builtin BLt) (Just "<") False 4) a b)  (.<=.) ::     (Columnable a, Ord a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool-(.<=.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (<=) "leq" (Just "<=") False 4) a b)+(.<=.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (<=) (Builtin BLeq) (Just "<=") False 4) a b)  (.>=.) ::     (Columnable a, Ord a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool-(.>=.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (>=) "geq" (Just ">=") False 4) a b)+(.>=.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (>=) (Builtin BGeq) (Just ">=") False 4) a b)  (.>.) ::     (Columnable a, Ord a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool-(.>.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (>) "gt" (Just ">") False 4) a b)+(.>.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (>) (Builtin BGt) (Just ">") False 4) a b)  -- Same-type arithmetic operators @@ -306,18 +306,18 @@ (.^^.) ::     (Columnable a, Columnable b, Fractional a, Integral b) =>     TExpr cols a -> TExpr cols b -> TExpr cols a-(.^^.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (^^) "pow" (Just ".^^.") False 8) a b)+(.^^.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (^^) (Builtin BPowFrac) (Just ".^^.") False 8) a b)  (.^.) ::     (Columnable a, Columnable b, Num a, Integral b) =>     TExpr cols a -> TExpr cols b -> TExpr cols a-(.^.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (^) "pow" (Just ".^.") False 8) a b)+(.^.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (^) (Builtin BPow) (Just ".^.") False 8) a b)  (.&&.) :: TExpr cols Bool -> TExpr cols Bool -> TExpr cols Bool-(.&&.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (&&) "and" (Just ".&&.") True 3) a b)+(.&&.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (&&) (Builtin BAnd) (Just ".&&.") True 3) a b)  (.||.) :: TExpr cols Bool -> TExpr cols Bool -> TExpr cols Bool-(.||.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (||) "or" (Just ".||.") True 2) a b)+(.||.) (TExpr a) (TExpr b) = TExpr (Binary (MkBinaryOp (||) (Builtin BOr) (Just ".||.") True 2) a b)  -- | Nullable-aware logical AND; @Maybe Bool@ when either operand is nullable. (.&&) ::@@ -326,7 +326,8 @@     TExpr cols b ->     TExpr cols (NullCmpResult a b) (.&&) (TExpr a) (TExpr b) =-    TExpr (Binary (MkBinaryOp (nullCmpOp (&&)) "nulland" (Just ".&&") True 3) a b)+    TExpr+        (Binary (MkBinaryOp (nullCmpOp (&&)) (Builtin BNullAnd) (Just ".&&") True 3) a b)  -- | Nullable-aware logical OR; @Maybe Bool@ when either operand is nullable. (.||) ::@@ -335,7 +336,8 @@     TExpr cols b ->     TExpr cols (NullCmpResult a b) (.||) (TExpr a) (TExpr b) =-    TExpr (Binary (MkBinaryOp (nullCmpOp (||)) "nullor" (Just ".||") True 2) a b)+    TExpr+        (Binary (MkBinaryOp (nullCmpOp (||)) (Builtin BNullOr) (Just ".||") True 2) a b)  infixl 6 .+, .- infixl 7 .*, ./@@ -352,7 +354,7 @@ (.+) (TExpr a) (TExpr b) =     TExpr         ( Binary-            (MkBinaryOp (applyNull2 (widenArithOp (+))) "nulladd" (Just "+") True 6)+            (MkBinaryOp (applyNull2 (widenArithOp (+))) (Builtin BNullAdd) (Just "+") True 6)             a             b         )@@ -369,7 +371,7 @@ (.-) (TExpr a) (TExpr b) =     TExpr         ( Binary-            (MkBinaryOp (applyNull2 (widenArithOp (-))) "nullsub" (Just "-") False 6)+            (MkBinaryOp (applyNull2 (widenArithOp (-))) (Builtin BNullSub) (Just "-") False 6)             a             b         )@@ -386,7 +388,7 @@ (.*) (TExpr a) (TExpr b) =     TExpr         ( Binary-            (MkBinaryOp (applyNull2 (widenArithOp (*))) "nullmul" (Just "*") True 7)+            (MkBinaryOp (applyNull2 (widenArithOp (*))) (Builtin BNullMul) (Just "*") True 7)             a             b         )@@ -403,7 +405,7 @@ (./) (TExpr a) (TExpr b) =     TExpr         ( Binary-            (MkBinaryOp (applyNull2 (divArithOp (/))) "nulldiv" (Just "/") False 7)+            (MkBinaryOp (applyNull2 (divArithOp (/))) (Builtin BNullDiv) (Just "/") False 7)             a             b         )@@ -420,7 +422,12 @@     ) =>     TExpr cols a -> TExpr cols b -> TExpr cols a (.^^) (TExpr a) (TExpr b) =-    TExpr (Binary (MkBinaryOp (applyNull2 (^^)) "pow" (Just ".^^") False 8) a b)+    TExpr+        ( Binary+            (MkBinaryOp (applyNull2 (^^)) (Builtin BPowFrac) (Just ".^^") False 8)+            a+            b+        )  -- | Nullable-aware exponentiation (num base, integral exponent). (.^) ::@@ -434,7 +441,8 @@     ) =>     TExpr cols a -> TExpr cols b -> TExpr cols a (.^) (TExpr a) (TExpr b) =-    TExpr (Binary (MkBinaryOp (applyNull2 (^)) "pow" (Just ".^") False 8) a b)+    TExpr+        (Binary (MkBinaryOp (applyNull2 (^)) (Builtin BPow) (Just ".^") False 8) a b)  -- | Nullable-aware equality that widens numeric operands to a common type. (.==) ::@@ -447,7 +455,11 @@     TExpr cols (NullCmpResult a b) (.==) (TExpr a) (TExpr b) =     TExpr-        (Binary (MkBinaryOp (applyNull2 (widenCmpOp (==))) "eq" (Just "==") True 4) a b)+        ( Binary+            (MkBinaryOp (applyNull2 (widenCmpOp (==))) (Builtin BEq) (Just "==") True 4)+            a+            b+        )  -- | Nullable-aware inequality. Widens numeric operands to their common type. (./=) ::@@ -460,7 +472,11 @@     TExpr cols (NullCmpResult a b) (./=) (TExpr a) (TExpr b) =     TExpr-        (Binary (MkBinaryOp (applyNull2 (widenCmpOp (/=))) "neq" (Just "/=") True 4) a b)+        ( Binary+            (MkBinaryOp (applyNull2 (widenCmpOp (/=))) (Builtin BNeq) (Just "/=") True 4)+            a+            b+        )  -- | Nullable-aware less-than. Widens numeric operands to their common type. (.<) ::@@ -473,7 +489,11 @@     TExpr cols (NullCmpResult a b) (.<) (TExpr a) (TExpr b) =     TExpr-        (Binary (MkBinaryOp (applyNull2 (widenCmpOp (<))) "lt" (Just "<") False 4) a b)+        ( Binary+            (MkBinaryOp (applyNull2 (widenCmpOp (<))) (Builtin BLt) (Just "<") False 4)+            a+            b+        )  -- | Nullable-aware <=. Widens numeric operands to their common type. (.<=) ::@@ -486,7 +506,11 @@     TExpr cols (NullCmpResult a b) (.<=) (TExpr a) (TExpr b) =     TExpr-        (Binary (MkBinaryOp (applyNull2 (widenCmpOp (<=))) "leq" (Just "<=") False 4) a b)+        ( Binary+            (MkBinaryOp (applyNull2 (widenCmpOp (<=))) (Builtin BLeq) (Just "<=") False 4)+            a+            b+        )  -- | Nullable-aware >=. Widens numeric operands to their common type. (.>=) ::@@ -499,7 +523,11 @@     TExpr cols (NullCmpResult a b) (.>=) (TExpr a) (TExpr b) =     TExpr-        (Binary (MkBinaryOp (applyNull2 (widenCmpOp (>=))) "geq" (Just ">=") False 4) a b)+        ( Binary+            (MkBinaryOp (applyNull2 (widenCmpOp (>=))) (Builtin BGeq) (Just ">=") False 4)+            a+            b+        )  -- | Nullable-aware greater-than. Widens numeric operands to their common type. (.>) ::@@ -512,10 +540,14 @@     TExpr cols (NullCmpResult a b) (.>) (TExpr a) (TExpr b) =     TExpr-        (Binary (MkBinaryOp (applyNull2 (widenCmpOp (>))) "gt" (Just ">") False 4) a b)+        ( Binary+            (MkBinaryOp (applyNull2 (widenCmpOp (>))) (Builtin BGt) (Just ">") False 4)+            a+            b+        )  not :: TExpr cols Bool -> TExpr cols Bool-not (TExpr e) = TExpr (Unary (MkUnaryOp Prelude.not "not" (Just "!")) e)+not (TExpr e) = TExpr (F.not e)  sum :: (Columnable a, Num a) => TExpr cols a -> TExpr cols a sum (TExpr e) = TExpr (F.sum e)@@ -552,43 +584,23 @@ castExpr ::     forall b cols src.     (Columnable b, Columnable src, Read b) => TExpr cols src -> TExpr cols (Maybe b)-castExpr (TExpr e) =-    TExpr-        (CastExprWith @b @(Maybe b) @src "castExpr" (either (const Nothing) Just) e)+castExpr (TExpr e) = TExpr (F.castExpr @b e)  castExprWithDefault ::     forall b cols src.     (Columnable b, Columnable src, Read b) => b -> TExpr cols src -> TExpr cols b-castExprWithDefault def (TExpr e) =-    TExpr-        ( CastExprWith @b @b @src-            ("castExprWithDefault:" <> T.pack (show def))-            (fromRight def)-            e-        )+castExprWithDefault def (TExpr e) = TExpr (F.castExprWithDefault def e)  castExprEither ::     forall b cols src.     (Columnable b, Columnable src, Read b) =>     TExpr cols src -> TExpr cols (Either T.Text b)-castExprEither (TExpr e) =-    TExpr-        ( CastExprWith @b @(Either T.Text b) @src-            "castExprEither"-            (either (Left . T.pack) Right)-            e-        )+castExprEither (TExpr e) = TExpr (F.castExprEither @b e)  unsafeCastExpr ::     forall b cols src.     (Columnable b, Columnable src, Read b) => TExpr cols src -> TExpr cols b-unsafeCastExpr (TExpr e) =-    TExpr-        ( CastExprWith @b @b @src-            "unsafeCastExpr"-            (fromRight (error "unsafeCastExpr: unexpected Nothing in column"))-            e-        )+unsafeCastExpr (TExpr e) = TExpr (F.unsafeCastExpr @b e)  toDouble :: (Columnable a, Real a) => TExpr cols a -> TExpr cols Double toDouble (TExpr e) = TExpr (F.toDouble e)
src/DataFrame/Typed/Expr/Extra.hs view
@@ -1,9 +1,31 @@+{-# LANGUAGE AllowAmbiguousTypes #-}+{-# LANGUAGE DataKinds #-} {-# LANGUAGE FlexibleContexts #-}-{-# LANGUAGE MonoLocalBinds #-} {-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}+{-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE TypeOperators #-}  -- | Typed counterparts of the remaining "DataFrame.Functions" combinators. module DataFrame.Typed.Expr.Extra (+    cast,+    castWithDefault,+    castEither,+    unsafeCast,+    eq,+    lt,+    gt,+    leq,+    geq,+    and,+    or,+    countDistinct,+    topK,+    bottomK,+    isNull,+    isNotNull,+    contains,+    length,     div,     mod,     mode,@@ -40,14 +62,115 @@     bind, ) where +import Data.Proxy (Proxy (..)) import qualified Data.Text as T import Data.Time (Day, ParseTime) import qualified Data.Vector.Unboxed as VU-import Prelude hiding (div, max, min, mod)+import GHC.TypeLits (KnownSymbol, Symbol, symbolVal)+import Prelude hiding (and, div, length, max, min, mod, or)  import qualified DataFrame.Functions as F import DataFrame.Internal.Column (Columnable)+import DataFrame.Internal.Expression.Operators.Nullable (BaseType)+import DataFrame.Typed.Schema (AssertPresent) import DataFrame.Typed.Types (TExpr (..))++colName :: forall (name :: Symbol). (KnownSymbol name) => T.Text+colName = T.pack (symbolVal (Proxy @name))++-- | Parse a schema-checked column as @a@; failures become 'Nothing'.+cast ::+    forall (name :: Symbol) a cols.+    (KnownSymbol name, AssertPresent name cols, Columnable a, Read a) =>+    TExpr cols (Maybe a)+cast = TExpr (F.cast @a (colName @name))++-- | Parse a schema-checked column as @a@, with a default for failures.+castWithDefault ::+    forall (name :: Symbol) a cols.+    (KnownSymbol name, AssertPresent name cols, Columnable a, Read a) =>+    a -> TExpr cols a+castWithDefault d = TExpr (F.castWithDefault d (colName @name))++-- | Parse a schema-checked column as @a@, keeping failures as 'Left'.+castEither ::+    forall (name :: Symbol) a cols.+    (KnownSymbol name, AssertPresent name cols, Columnable a, Read a) =>+    TExpr cols (Either T.Text a)+castEither = TExpr (F.castEither @a (colName @name))++-- | Parse a schema-checked column as @a@, erroring on failures.+unsafeCast ::+    forall (name :: Symbol) a cols.+    (KnownSymbol name, AssertPresent name cols, Columnable a, Read a) =>+    TExpr cols a+unsafeCast = TExpr (F.unsafeCast @a (colName @name))++-- | Equality.+eq ::+    (Columnable a, Eq a, a ~ BaseType a) =>+    TExpr cols a -> TExpr cols a -> TExpr cols Bool+eq (TExpr a) (TExpr b) = TExpr (F.eq a b)++-- | Less than.+lt ::+    (Columnable a, Ord a, a ~ BaseType a) =>+    TExpr cols a -> TExpr cols a -> TExpr cols Bool+lt (TExpr a) (TExpr b) = TExpr (F.lt a b)++-- | Greater than.+gt ::+    (Columnable a, Ord a, a ~ BaseType a) =>+    TExpr cols a -> TExpr cols a -> TExpr cols Bool+gt (TExpr a) (TExpr b) = TExpr (F.gt a b)++-- | Less than or equal.+leq ::+    (Columnable a, Ord a, a ~ BaseType a) =>+    TExpr cols a -> TExpr cols a -> TExpr cols Bool+leq (TExpr a) (TExpr b) = TExpr (F.leq a b)++-- | Greater than or equal.+geq ::+    (Columnable a, Ord a, a ~ BaseType a) =>+    TExpr cols a -> TExpr cols a -> TExpr cols Bool+geq (TExpr a) (TExpr b) = TExpr (F.geq a b)++-- | Logical and.+and :: TExpr cols Bool -> TExpr cols Bool -> TExpr cols Bool+and (TExpr a) (TExpr b) = TExpr (F.and a b)++-- | Logical or.+or :: TExpr cols Bool -> TExpr cols Bool -> TExpr cols Bool+or (TExpr a) (TExpr b) = TExpr (F.or a b)++-- | Number of distinct values (aggregation).+countDistinct :: (Columnable a, Ord a) => TExpr cols a -> TExpr cols Int+countDistinct (TExpr e) = TExpr (F.countDistinct e)++-- | The k largest values per group (aggregation).+topK :: (Columnable a, Ord a) => Int -> TExpr cols a -> TExpr cols [a]+topK k (TExpr e) = TExpr (F.topK k e)++-- | The k smallest values per group (aggregation).+bottomK :: (Columnable a, Ord a) => Int -> TExpr cols a -> TExpr cols [a]+bottomK k (TExpr e) = TExpr (F.bottomK k e)++-- | SQL spelling of 'isNothing'.+isNull :: (Columnable a) => TExpr cols (Maybe a) -> TExpr cols Bool+isNull (TExpr e) = TExpr (F.isNull e)++-- | SQL spelling of 'isJust'.+isNotNull :: (Columnable a) => TExpr cols (Maybe a) -> TExpr cols Bool+isNotNull (TExpr e) = TExpr (F.isNotNull e)++-- | Whether the text contains the literal substring.+contains :: T.Text -> TExpr cols T.Text -> TExpr cols Bool+contains needle (TExpr e) = TExpr (F.contains needle e)++-- | Character count.+length :: TExpr cols T.Text -> TExpr cols Int+length (TExpr e) = TExpr (F.length e)  -- | Integer division. div ::
+ src/DataFrame/Typed/Functions.hs view
@@ -0,0 +1,6 @@+-- | Typed expression combinators; the typed "DataFrame.Functions".+module DataFrame.Typed.Functions (+    module DataFrame.Typed.Expr,+) where++import DataFrame.Typed.Expr
src/DataFrame/Typed/Join.hs view
@@ -1,4 +1,5 @@ {-# LANGUAGE AllowAmbiguousTypes #-}+{-# LANGUAGE ConstraintKinds #-} {-# LANGUAGE DataKinds #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-}@@ -6,20 +7,32 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE TypeOperators #-}  module DataFrame.Typed.Join (     -- * Typed joins+    join,+    joinOn,+    JoinSchema,+    JoinOnSchema,+    JoinKeys,+    KnownJoinType (..),+    Fsts,+    SwapPairs,     innerJoin,     leftJoin,     rightJoin,     fullOuterJoin, ) where +import Data.Kind (Type) import GHC.TypeLits (Symbol) +import DataFrame.Operations.Join (JoinType (..)) import qualified DataFrame.Operations.Join as DJ  import DataFrame.Typed.Freeze (unsafeFreeze)+import DataFrame.Typed.Operations (AllKnownPairs (..)) import DataFrame.Typed.Schema import DataFrame.Typed.Types (TypedDataFrame (..)) @@ -35,7 +48,7 @@     TypedDataFrame right ->     TypedDataFrame (InnerJoinSchema keys left right) innerJoin (TDF l) (TDF r) =-    unsafeFreeze (DJ.innerJoin keyNames r l)+    unsafeFreeze (DJ.innerJoin keyNames l r)   where     keyNames = symbolVals @keys @@ -83,6 +96,74 @@     TypedDataFrame right ->     TypedDataFrame (FullOuterJoinSchema keys left right) fullOuterJoin (TDF l) (TDF r) =-    unsafeFreeze (DJ.fullOuterJoin keyNames r l)+    unsafeFreeze (DJ.fullOuterJoin keyNames l r)   where     keyNames = symbolVals @keys++-- | Keys present on both sides with matching types.+type JoinKeys keys left right =+    ( AllKnownSymbol keys+    , AssertAllPresent keys left+    , AssertAllPresent keys right+    , AssertKeyTypesMatch keys left right+    )++-- | Result schema of a @jt@ join on @keys@.+type family+    JoinSchema+        (jt :: JoinType)+        (keys :: [Symbol])+        (left :: [(Symbol, Type)])+        (right :: [(Symbol, Type)]) ::+        [(Symbol, Type)]+    where+    JoinSchema 'INNER keys left right = InnerJoinSchema keys left right+    JoinSchema 'LEFT keys left right = LeftJoinSchema keys left right+    JoinSchema 'RIGHT keys left right = RightJoinSchema keys left right+    JoinSchema 'FULL_OUTER keys left right = FullOuterJoinSchema keys left right++-- | Reflect a promoted 'JoinType' to its value.+class KnownJoinType (jt :: JoinType) where+    joinTypeVal :: JoinType++instance KnownJoinType 'INNER where joinTypeVal = INNER+instance KnownJoinType 'LEFT where joinTypeVal = LEFT+instance KnownJoinType 'RIGHT where joinTypeVal = RIGHT+instance KnownJoinType 'FULL_OUTER where joinTypeVal = FULL_OUTER++-- | Typed counterpart of untyped @join@: @join \@'INNER \@'["key"] l r@.+join ::+    forall (jt :: JoinType) (keys :: [Symbol]) left right.+    (KnownJoinType jt, JoinKeys keys left right) =>+    TypedDataFrame left ->+    TypedDataFrame right ->+    TypedDataFrame (JoinSchema jt keys left right)+join (TDF l) (TDF r) =+    unsafeFreeze (DJ.join (joinTypeVal @jt) (symbolVals @keys) l r)++-- | Left names of @(leftKey, rightKey)@ pairs.+type family Fsts (pairs :: [(Symbol, Symbol)]) :: [Symbol] where+    Fsts '[] = '[]+    Fsts ('(l, _) ': rest) = l ': Fsts rest++-- | Flip each pair, turning @(left, right)@ into a right-to-left rename.+type family SwapPairs (pairs :: [(Symbol, Symbol)]) :: [(Symbol, Symbol)] where+    SwapPairs '[] = '[]+    SwapPairs ('(l, r) ': rest) = '(r, l) ': SwapPairs rest++-- | Result schema of 'joinOn': right keys take their left names first.+type JoinOnSchema jt pairs left right =+    JoinSchema jt (Fsts pairs) left (RenameManyInSchema (SwapPairs pairs) right)++-- | Join on key pairs: @joinOn \@'INNER \@'[ '("id", "uid")] l r@.+joinOn ::+    forall (jt :: JoinType) (pairs :: [(Symbol, Symbol)]) left right.+    ( KnownJoinType jt+    , AllKnownPairs pairs+    , JoinKeys (Fsts pairs) left (RenameManyInSchema (SwapPairs pairs) right)+    ) =>+    TypedDataFrame left ->+    TypedDataFrame right ->+    TypedDataFrame (JoinOnSchema jt pairs left right)+joinOn (TDF l) (TDF r) =+    unsafeFreeze (DJ.joinOn (joinTypeVal @jt) (pairVals @pairs) l r)
src/DataFrame/Typed/Operations.hs view
@@ -11,7 +11,6 @@  module DataFrame.Typed.Operations (     -- * Schema-preserving operations-    filterWhere,     filter,     filterBy,     filterAllJust,@@ -19,6 +18,7 @@     filterNothing,     filterAllNothing,     sortBy,+    topBy,     take,     takeLast,     drop,@@ -31,6 +31,7 @@      -- * Schema-modifying operations     derive,+    deriveMany,     impute,     select,     exclude,@@ -47,6 +48,11 @@     valueCounts,     valueProportions, +    -- * Construction and folds+    empty,+    null,+    fold,+     -- * Metadata     dimensions,     nRows,@@ -61,14 +67,19 @@     intersect,     difference,     symmetricDifference,++    -- * Helpers+    AllKnownPairs (..),+    toUntypedSort, ) where +import qualified Data.List as L import Data.Proxy (Proxy (..)) import qualified Data.Text as T import qualified Data.Vector as V import GHC.TypeLits (KnownSymbol, Symbol, symbolVal) import System.Random (RandomGen)-import Prelude hiding (drop, filter, take)+import Prelude hiding (drop, filter, null, take)  import qualified DataFrame.Functions as DF import DataFrame.Internal.Column (Columnable)@@ -84,19 +95,19 @@  import DataFrame.Typed.Freeze (unsafeFreeze) import DataFrame.Typed.Schema-import DataFrame.Typed.Types (TExpr (..), TSortOrder (..), TypedDataFrame (..))+import DataFrame.Typed.Types (+    TAgg (..),+    TExpr (..),+    TSortOrder (..),+    TypedDataFrame (..),+    taggToNamedExprs,+ )  -- | Filter rows where a boolean expression evaluates to True.-filterWhere :: TExpr cols Bool -> TypedDataFrame cols -> TypedDataFrame cols-filterWhere (TExpr expr) (TDF df) = TDF (D.filterWhere expr df)+filter :: TExpr cols Bool -> TypedDataFrame cols -> TypedDataFrame cols+filter (TExpr expr) (TDF df) = TDF (D.filter expr df)  -- | Filter rows by applying a predicate to a typed expression.-filter ::-    (Columnable a) =>-    TExpr cols a -> (a -> Bool) -> TypedDataFrame cols -> TypedDataFrame cols-filter (TExpr expr) pred' (TDF df) = TDF (D.filter expr pred' df)---- | Filter rows by a predicate on a column expression (flipped argument order). filterBy ::     (Columnable a) =>     (a -> Bool) -> TExpr cols a -> TypedDataFrame cols -> TypedDataFrame cols@@ -135,11 +146,17 @@ -- | Sort by the given typed sort orders, validated against the schema. sortBy :: [TSortOrder cols] -> TypedDataFrame cols -> TypedDataFrame cols sortBy ords (TDF df) = TDF (D.sortBy (map toUntypedSort ords) df)-  where-    toUntypedSort :: TSortOrder cols -> D.SortOrder-    toUntypedSort (Asc (TExpr e)) = D.Asc e-    toUntypedSort (Desc (TExpr e)) = D.Desc e +-- | Rows @[offset, offset + count)@ of the frame sorted by the given orders.+topBy ::+    Int -> Int -> [TSortOrder cols] -> TypedDataFrame cols -> TypedDataFrame cols+topBy offset count ords (TDF df) =+    TDF (D.topBy offset count (map toUntypedSort ords) df)++toUntypedSort :: TSortOrder cols -> D.SortOrder+toUntypedSort (Asc (TExpr e)) = D.Asc e+toUntypedSort (Desc (TExpr e)) = D.Desc e+ -- | Take the first @n@ rows. take :: Int -> TypedDataFrame cols -> TypedDataFrame cols take n (TDF df) = TDF (D.take n df)@@ -191,6 +208,16 @@   where     colName = T.pack (symbolVal (Proxy @name)) +-- | Add several columns at once; build them with 'DataFrame.Typed.as'.+deriveMany ::+    forall new cols.+    (AssertAllAbsent (Reverse new) cols) =>+    (TAgg '[] cols '[] -> TAgg '[] cols new) ->+    TypedDataFrame cols ->+    TypedDataFrame (Append cols (Reverse new))+deriveMany build (TDF df) =+    unsafeFreeze (D.deriveMany (taggToNamedExprs (build TAggNil)) df)+ impute ::     forall name a cols.     ( KnownSymbol name@@ -335,6 +362,22 @@ symmetricDifference ::     TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols symmetricDifference (TDF a) (TDF b) = TDF (DS.symmetricDifference a b)++-- | The frame with no rows and no columns.+empty :: TypedDataFrame '[]+empty = TDF D.empty++-- | True when the frame has no rows or no columns.+null :: TypedDataFrame cols -> Bool+null (TDF df) = D.null df++-- | Thread the frame through a schema-preserving step for each element.+fold ::+    (a -> TypedDataFrame cols -> TypedDataFrame cols) ->+    [a] ->+    TypedDataFrame cols ->+    TypedDataFrame cols+fold f xs df = L.foldl' (flip f) df xs  dimensions :: TypedDataFrame cols -> (Int, Int) dimensions (TDF df) = D.dimensions df