dataframe-operations 1.0.1.1 → 1.1.0.1

raw patch · 6 files changed

+201/−54 lines, 6 filesdep −hashabledep ~dataframe-corePVP ok

version bump matches the API change (PVP)

Dependencies removed: hashable

Dependency ranges changed: dataframe-core

API changes (from Hackage documentation)

Files

• dataframe-operations.cabal +3/−3
• src/DataFrame/Functions.hs +1/−0
• src/DataFrame/Operations/Aggregation.hs +55/−49
• src/DataFrame/Operations/SetOps.hs +88/−0
• src/DataFrame/Typed/Expr.hs +21/−2
• src/DataFrame/Typed/Operations.hs +33/−0

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-operations-version:            1.0.1.1+version:            1.1.0.1  synopsis:           Column operations, expression DSL, and statistics for the dataframe ecosystem. description:@@ -16,7 +16,7 @@ license-file:       LICENSE author:             Michael Chavinda maintainer:         mschavinda@gmail.com-copyright:          (c) 2024-2025 Michael Chavinda+copyright:          (c) 2024-2026 Michael Chavinda category:           Data tested-with:        GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.3 || ==9.12.2 @@ -39,6 +39,7 @@                         DataFrame.Operations.Join                         DataFrame.Operations.Merge                         DataFrame.Operations.Permutation+                        DataFrame.Operations.SetOps                         DataFrame.Operations.Statistics                         DataFrame.Operations.Subset                         DataFrame.Operations.Transformations@@ -52,7 +53,6 @@                         containers >= 0.6.7 && < 0.9,                         dataframe-core ^>= 1.0,                         dataframe-parsing ^>= 1.0,-                        hashable >= 1.2 && < 2,                         random >= 1.2 && < 2,                         regex-tdfa >= 1.3.0 && < 2,                         text >= 2.0 && < 3,

@@ -1,3 +1,4 @@+{-# LANGUAGE DisambiguateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-}

@@ -21,11 +21,11 @@  import Control.Exception (throw) import Control.Monad-import Control.Monad.ST (runST)-import Data.Hashable+import Control.Monad.ST (ST, runST) import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) import DataFrame.Errors import DataFrame.Internal.Column (+    Bitmap,     Column (..),     TypedColumn (..),     atIndicesStable,@@ -39,6 +39,14 @@  ) import DataFrame.Internal.Expression import DataFrame.Internal.Grouping (buildRowToGroup, changingPoints, groupBy)+import DataFrame.Internal.Hash (+    fnvOffset,+    mixDouble,+    mixInt,+    mixShow,+    mixText,+    nullSalt,+ ) import DataFrame.Internal.Interpreter import DataFrame.Internal.Types import DataFrame.Operations.Core@@ -48,59 +56,28 @@ computeRowHashes :: [Int] -> DataFrame -> VU.Vector Int computeRowHashes indices df = runST $ do     let n = fst (dimensions df)-    mv <- VUM.new n+    mv <- VUM.replicate n fnvOffset      let selectedCols = map (columns df V.!) indices      forM_ selectedCols $ \case-        UnboxedColumn _ (v :: VU.Vector a) ->+        UnboxedColumn ubm (v :: VU.Vector a) ->             case testEquality (typeRep @a) (typeRep @Int) of-                Just Refl ->-                    VU.imapM_-                        ( \i (x :: Int) -> do-                            h <- VUM.unsafeRead mv i-                            VUM.unsafeWrite mv i (hashWithSalt h x)-                        )-                        v+                Just Refl -> hashKeyUnboxed mv ubm mixInt v                 Nothing ->                     case testEquality (typeRep @a) (typeRep @Double) of                         Just Refl ->-                            VU.imapM_-                                ( \i (d :: Double) -> do-                                    h <- VUM.unsafeRead mv i-                                    VUM.unsafeWrite mv i (hashWithSalt h (doubleToInt d))-                                )-                                v+                            hashKeyUnboxed mv ubm mixDouble v                         Nothing ->                             case sIntegral @a of                                 STrue ->-                                    VU.imapM_-                                        ( \i d -> do-                                            let x :: Int-                                                x = fromIntegral @a @Int d-                                            h <- VUM.unsafeRead mv i-                                            VUM.unsafeWrite mv i (hashWithSalt h x)-                                        )-                                        v+                                    hashKeyUnboxed mv ubm (\h d -> mixInt h (fromIntegral @a @Int d)) v                                 SFalse ->                                     case sFloating @a of                                         STrue ->-                                            VU.imapM_-                                                ( \i d -> do-                                                    let x :: Int-                                                        x = doubleToInt (realToFrac d :: Double)-                                                    h <- VUM.unsafeRead mv i-                                                    VUM.unsafeWrite mv i (hashWithSalt h x)-                                                )-                                                v+                                            hashKeyUnboxed mv ubm (\h d -> mixDouble h (realToFrac d :: Double)) v                                         SFalse ->-                                            VU.imapM_-                                                ( \i d -> do-                                                    let x = hash (show d)-                                                    h <- VUM.unsafeRead mv i-                                                    VUM.unsafeWrite mv i (hashWithSalt h x)-                                                )-                                                v+                                            hashKeyUnboxed mv ubm mixShow v         BoxedColumn bm (v :: V.Vector a) ->             case testEquality (typeRep @a) (typeRep @T.Text) of                 Just Refl ->@@ -108,26 +85,55 @@                         ( \i (t :: T.Text) -> do                             h <- VUM.unsafeRead mv i                             let h' = case bm of-                                    Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int)-                                    _ -> hashWithSalt h t+                                    Just bm' | not (bitmapTestBit bm' i) -> mixInt h nullSalt+                                    _ -> mixText h t                             VUM.unsafeWrite mv i h'                         )                         v                 Nothing ->                     V.imapM_                         ( \i d -> do-                            let x = case bm of-                                    Just bm' | not (bitmapTestBit bm' i) -> 0 :: Int-                                    _ -> hash (show d)                             h <- VUM.unsafeRead mv i-                            VUM.unsafeWrite mv i (hashWithSalt h x)+                            let h' = case bm of+                                    Just bm' | not (bitmapTestBit bm' i) -> mixInt h nullSalt+                                    _ -> mixShow h d+                            VUM.unsafeWrite mv i h'                         )                         v      VU.unsafeFreeze mv-  where-    doubleToInt :: Double -> Int-    doubleToInt = floor . (* 1000)++{- | Fold a value-mix over an unboxed key column into the running hash vector,+respecting the null bitmap (a null slot mixes 'nullSalt' instead of its+uninitialised stored value, so @Nothing@ never matches a present @Just x@).+The non-nullable case is the original tight loop. @INLINE@d to specialise per call.+-}+hashKeyUnboxed ::+    (VU.Unbox a) =>+    VUM.MVector s Int ->+    Maybe Bitmap ->+    (Int -> a -> Int) ->+    VU.Vector a ->+    ST s ()+hashKeyUnboxed mv ubm mix v = case ubm of+    Nothing ->+        VU.imapM_+            ( \i x -> do+                h <- VUM.unsafeRead mv i+                VUM.unsafeWrite mv i (mix h x)+            )+            v+    Just bm ->+        VU.imapM_+            ( \i x -> do+                h <- VUM.unsafeRead mv i+                VUM.unsafeWrite+                    mv+                    i+                    (if bitmapTestBit bm i then mix h x else mixInt h nullSalt)+            )+            v+{-# INLINE hashKeyUnboxed #-}  {- | Aggregate a grouped dataframe using the expressions given. All ungrouped columns will be dropped.

@@ -0,0 +1,88 @@+{-# LANGUAGE ScopedTypeVariables #-}++{- |+Module      : DataFrame.Operations.SetOps+Description : Set-theoretic ("topos") row operations.++These treat a 'DataFrame' as a /set/ of rows and implement the subobject+lattice from relational algebra: 'union', 'intersect', 'difference', and+'symmetricDifference'. Every result is deduplicated, so each operation has the+schema-preserving shape @DataFrame -> DataFrame -> DataFrame@.++Row equality is the same hash-based notion used by 'distinct' (see+"DataFrame.Operations.Aggregation"), so these operations and 'distinct' agree+on what "the same row" means. Both inputs are expected to share a schema; the+typed layer ('DataFrame.Typed') enforces that statically.+-}+module DataFrame.Operations.SetOps (+    union,+    intersect,+    difference,+    symmetricDifference,+) where++import Prelude hiding (filter)++import qualified Data.Vector.Unboxed as VU++import DataFrame.Internal.DataFrame (+    DataFrame (..),+    GroupedDataFrame (..),+    columnNames,+ )+import DataFrame.Operations.Aggregation (distinct, groupBy, selectIndices)+import DataFrame.Operations.Merge ()++{- | All rows that appear in either dataframe, deduplicated.++@union a b@ is @distinct (a <> b)@: the set union of the two row sets.+-}+union :: DataFrame -> DataFrame -> DataFrame+union a b = distinct (a <> b)++{- | Rows that appear in both dataframes, deduplicated.++A row survives iff an equal row is present in each input.+-}+intersect :: DataFrame -> DataFrame -> DataFrame+intersect = setOp (&&)++{- | Rows present in the left dataframe but absent from the right, deduplicated+(the relational @EXCEPT@; the subobject complement of @a@ by @b@).+-}+difference :: DataFrame -> DataFrame -> DataFrame+difference = setOp (\inLeft inRight -> inLeft && not inRight)++{- | Rows present in exactly one of the two dataframes, deduplicated.++@symmetricDifference a b@ is @union (difference a b) (difference b a)@.+-}+symmetricDifference :: DataFrame -> DataFrame -> DataFrame+symmetricDifference a b = difference a b `union` difference b a++{- | Core engine for 'intersect' and 'difference'.++Concatenate the inputs, group by every column, and decide each group from+whether it has a member on the left side (original-row index @< nRows a@) and/or+the right side. The first member of a qualifying group is emitted as the+representative row, which keeps the result deduplicated and, because group+members come out in ascending row order, prefers the left dataframe's row.+-}+setOp :: (Bool -> Bool -> Bool) -> DataFrame -> DataFrame -> DataFrame+setOp keep a b =+    selectIndices (VU.fromList chosen) combined+  where+    leftRows = fst (dataframeDimensions a)+    combined = a <> b+    Grouped _ _ vis offs _ = groupBy (columnNames combined) combined+    nGroups = VU.length offs - 1+    chosen =+        [ VU.head members+        | k <- [0 .. nGroups - 1]+        , let s = VU.unsafeIndex offs k+              e = VU.unsafeIndex offs (k + 1)+              members = VU.slice s (e - s) vis+              inLeft = VU.any (< leftRows) members+              inRight = VU.any (>= leftRows) members+        , keep inLeft inRight+        ]

@@ -1,5 +1,6 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE DataKinds #-}+{-# LANGUAGE DisambiguateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-}@@ -128,14 +129,15 @@ import Data.Proxy (Proxy (..)) import Data.String (IsString (..)) import qualified Data.Text as T+import GHC.OverloadedLabels (IsLabel (..)) import GHC.TypeLits (KnownSymbol, Symbol, symbolVal)  import qualified DataFrame.Functions as F import DataFrame.Internal.Column (Columnable) import DataFrame.Internal.Expression (-    BinaryOp (..),+    BinUDF (..),     Expr (..),-    UnaryOp (..),+    UnUDF (..),  ) import DataFrame.Internal.Nullable (     BaseType,@@ -185,6 +187,23 @@     ) =>     TExpr cols a col = TExpr (Col (T.pack (symbolVal (Proxy @name))))++{- | Use a column name as an @OverloadedLabels@ label: @#age@ is sugar for+@col \@\"age\"@. Enable @OverloadedLabels@ at the use site.++@+adults = filterWhere (#age .>=. lit 18) people+@+-}+instance+    ( KnownSymbol name+    , a ~ SafeLookup name cols+    , Columnable a+    , AssertPresent name cols+    ) =>+    IsLabel name (TExpr cols a)+    where+    fromLabel = col @name  {- | Create a literal expression. Valid for any schema since it references no columns.

@@ -50,6 +50,12 @@      -- * Vertical merge     append,++    -- * Set algebra (topos operations)+    union,+    intersect,+    difference,+    symmetricDifference, ) where  import Data.Proxy (Proxy (..))@@ -67,6 +73,7 @@ import qualified DataFrame.Operations.Core as D import DataFrame.Operations.Merge () import qualified DataFrame.Operations.Permutation as D+import qualified DataFrame.Operations.SetOps as DS import qualified DataFrame.Operations.Subset as D import qualified DataFrame.Operations.Transformations as D @@ -340,6 +347,32 @@ -- | Vertically merge two DataFrames with the same schema. append :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols append (TDF a) (TDF b) = TDF (a <> b)++-------------------------------------------------------------------------------+-- Set algebra (topos operations)+--+-- Each treats a DataFrame as a /set/ of rows and is schema-preserving:+-- the output type equals the input type; only which rows are present changes.+-------------------------------------------------------------------------------++-- | Rows appearing in either DataFrame, deduplicated (set union).+union :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols+union (TDF a) (TDF b) = TDF (DS.union a b)++-- | Rows appearing in both DataFrames, deduplicated (set intersection).+intersect :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols+intersect (TDF a) (TDF b) = TDF (DS.intersect a b)++{- | Rows in the left DataFrame but not the right, deduplicated+(relational @EXCEPT@; the subobject complement).+-}+difference :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols+difference (TDF a) (TDF b) = TDF (DS.difference a b)++-- | Rows in exactly one of the two DataFrames, deduplicated.+symmetricDifference ::+    TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols+symmetricDifference (TDF a) (TDF b) = TDF (DS.symmetricDifference a b)  ------------------------------------------------------------------------------- -- Metadata (pass-through)