diff --git a/dataframe-operations.cabal b/dataframe-operations.cabal --- a/dataframe-operations.cabal +++ b/dataframe-operations.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe-operations -version: 1.0.1.1 +version: 1.1.0.1 synopsis: Column operations, expression DSL, and statistics for the dataframe ecosystem. description: @@ -16,7 +16,7 @@ license-file: LICENSE author: Michael Chavinda maintainer: mschavinda@gmail.com -copyright: (c) 2024-2025 Michael Chavinda +copyright: (c) 2024-2026 Michael Chavinda category: Data tested-with: GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.3 || ==9.12.2 @@ -39,6 +39,7 @@ DataFrame.Operations.Join DataFrame.Operations.Merge DataFrame.Operations.Permutation + DataFrame.Operations.SetOps DataFrame.Operations.Statistics DataFrame.Operations.Subset DataFrame.Operations.Transformations @@ -52,7 +53,6 @@ containers >= 0.6.7 && < 0.9, dataframe-core ^>= 1.0, dataframe-parsing ^>= 1.0, - hashable >= 1.2 && < 2, random >= 1.2 && < 2, regex-tdfa >= 1.3.0 && < 2, text >= 2.0 && < 3, diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs --- a/src/DataFrame/Functions.hs +++ b/src/DataFrame/Functions.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE DisambiguateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-} diff --git a/src/DataFrame/Operations/Aggregation.hs b/src/DataFrame/Operations/Aggregation.hs --- a/src/DataFrame/Operations/Aggregation.hs +++ b/src/DataFrame/Operations/Aggregation.hs @@ -21,11 +21,11 @@ import Control.Exception (throw) import Control.Monad -import Control.Monad.ST (runST) -import Data.Hashable +import Control.Monad.ST (ST, runST) import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) import DataFrame.Errors import DataFrame.Internal.Column ( + Bitmap, Column (..), TypedColumn (..), atIndicesStable, @@ -39,6 +39,14 @@ ) import DataFrame.Internal.Expression import DataFrame.Internal.Grouping (buildRowToGroup, changingPoints, groupBy) +import DataFrame.Internal.Hash ( + fnvOffset, + mixDouble, + mixInt, + mixShow, + mixText, + nullSalt, + ) import DataFrame.Internal.Interpreter import DataFrame.Internal.Types import DataFrame.Operations.Core @@ -48,59 +56,28 @@ computeRowHashes :: [Int] -> DataFrame -> VU.Vector Int computeRowHashes indices df = runST $ do let n = fst (dimensions df) - mv <- VUM.new n + mv <- VUM.replicate n fnvOffset let selectedCols = map (columns df V.!) indices forM_ selectedCols $ \case - UnboxedColumn _ (v :: VU.Vector a) -> + UnboxedColumn ubm (v :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of - Just Refl -> - VU.imapM_ - ( \i (x :: Int) -> do - h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h x) - ) - v + Just Refl -> hashKeyUnboxed mv ubm mixInt v Nothing -> case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> - VU.imapM_ - ( \i (d :: Double) -> do - h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h (doubleToInt d)) - ) - v + hashKeyUnboxed mv ubm mixDouble v Nothing -> case sIntegral @a of STrue -> - VU.imapM_ - ( \i d -> do - let x :: Int - x = fromIntegral @a @Int d - h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h x) - ) - v + hashKeyUnboxed mv ubm (\h d -> mixInt h (fromIntegral @a @Int d)) v SFalse -> case sFloating @a of STrue -> - VU.imapM_ - ( \i d -> do - let x :: Int - x = doubleToInt (realToFrac d :: Double) - h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h x) - ) - v + hashKeyUnboxed mv ubm (\h d -> mixDouble h (realToFrac d :: Double)) v SFalse -> - VU.imapM_ - ( \i d -> do - let x = hash (show d) - h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h x) - ) - v + hashKeyUnboxed mv ubm mixShow v BoxedColumn bm (v :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> @@ -108,26 +85,55 @@ ( \i (t :: T.Text) -> do h <- VUM.unsafeRead mv i let h' = case bm of - Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int) - _ -> hashWithSalt h t + Just bm' | not (bitmapTestBit bm' i) -> mixInt h nullSalt + _ -> mixText h t VUM.unsafeWrite mv i h' ) v Nothing -> V.imapM_ ( \i d -> do - let x = case bm of - Just bm' | not (bitmapTestBit bm' i) -> 0 :: Int - _ -> hash (show d) h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h x) + let h' = case bm of + Just bm' | not (bitmapTestBit bm' i) -> mixInt h nullSalt + _ -> mixShow h d + VUM.unsafeWrite mv i h' ) v VU.unsafeFreeze mv - where - doubleToInt :: Double -> Int - doubleToInt = floor . (* 1000) + +{- | Fold a value-mix over an unboxed key column into the running hash vector, +respecting the null bitmap (a null slot mixes 'nullSalt' instead of its +uninitialised stored value, so @Nothing@ never matches a present @Just x@). +The non-nullable case is the original tight loop. @INLINE@d to specialise per call. +-} +hashKeyUnboxed :: + (VU.Unbox a) => + VUM.MVector s Int -> + Maybe Bitmap -> + (Int -> a -> Int) -> + VU.Vector a -> + ST s () +hashKeyUnboxed mv ubm mix v = case ubm of + Nothing -> + VU.imapM_ + ( \i x -> do + h <- VUM.unsafeRead mv i + VUM.unsafeWrite mv i (mix h x) + ) + v + Just bm -> + VU.imapM_ + ( \i x -> do + h <- VUM.unsafeRead mv i + VUM.unsafeWrite + mv + i + (if bitmapTestBit bm i then mix h x else mixInt h nullSalt) + ) + v +{-# INLINE hashKeyUnboxed #-} {- | Aggregate a grouped dataframe using the expressions given. All ungrouped columns will be dropped. diff --git a/src/DataFrame/Operations/SetOps.hs b/src/DataFrame/Operations/SetOps.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Operations/SetOps.hs @@ -0,0 +1,88 @@ +{-# LANGUAGE ScopedTypeVariables #-} + +{- | +Module : DataFrame.Operations.SetOps +Description : Set-theoretic ("topos") row operations. + +These treat a 'DataFrame' as a /set/ of rows and implement the subobject +lattice from relational algebra: 'union', 'intersect', 'difference', and +'symmetricDifference'. Every result is deduplicated, so each operation has the +schema-preserving shape @DataFrame -> DataFrame -> DataFrame@. + +Row equality is the same hash-based notion used by 'distinct' (see +"DataFrame.Operations.Aggregation"), so these operations and 'distinct' agree +on what "the same row" means. Both inputs are expected to share a schema; the +typed layer ('DataFrame.Typed') enforces that statically. +-} +module DataFrame.Operations.SetOps ( + union, + intersect, + difference, + symmetricDifference, +) where + +import Prelude hiding (filter) + +import qualified Data.Vector.Unboxed as VU + +import DataFrame.Internal.DataFrame ( + DataFrame (..), + GroupedDataFrame (..), + columnNames, + ) +import DataFrame.Operations.Aggregation (distinct, groupBy, selectIndices) +import DataFrame.Operations.Merge () + +{- | All rows that appear in either dataframe, deduplicated. + +@union a b@ is @distinct (a <> b)@: the set union of the two row sets. +-} +union :: DataFrame -> DataFrame -> DataFrame +union a b = distinct (a <> b) + +{- | Rows that appear in both dataframes, deduplicated. + +A row survives iff an equal row is present in each input. +-} +intersect :: DataFrame -> DataFrame -> DataFrame +intersect = setOp (&&) + +{- | Rows present in the left dataframe but absent from the right, deduplicated +(the relational @EXCEPT@; the subobject complement of @a@ by @b@). +-} +difference :: DataFrame -> DataFrame -> DataFrame +difference = setOp (\inLeft inRight -> inLeft && not inRight) + +{- | Rows present in exactly one of the two dataframes, deduplicated. + +@symmetricDifference a b@ is @union (difference a b) (difference b a)@. +-} +symmetricDifference :: DataFrame -> DataFrame -> DataFrame +symmetricDifference a b = difference a b `union` difference b a + +{- | Core engine for 'intersect' and 'difference'. + +Concatenate the inputs, group by every column, and decide each group from +whether it has a member on the left side (original-row index @< nRows a@) and/or +the right side. The first member of a qualifying group is emitted as the +representative row, which keeps the result deduplicated and, because group +members come out in ascending row order, prefers the left dataframe's row. +-} +setOp :: (Bool -> Bool -> Bool) -> DataFrame -> DataFrame -> DataFrame +setOp keep a b = + selectIndices (VU.fromList chosen) combined + where + leftRows = fst (dataframeDimensions a) + combined = a <> b + Grouped _ _ vis offs _ = groupBy (columnNames combined) combined + nGroups = VU.length offs - 1 + chosen = + [ VU.head members + | k <- [0 .. nGroups - 1] + , let s = VU.unsafeIndex offs k + e = VU.unsafeIndex offs (k + 1) + members = VU.slice s (e - s) vis + inLeft = VU.any (< leftRows) members + inRight = VU.any (>= leftRows) members + , keep inLeft inRight + ] diff --git a/src/DataFrame/Typed/Expr.hs b/src/DataFrame/Typed/Expr.hs --- a/src/DataFrame/Typed/Expr.hs +++ b/src/DataFrame/Typed/Expr.hs @@ -1,5 +1,6 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE DataKinds #-} +{-# LANGUAGE DisambiguateRecordFields #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-} @@ -128,14 +129,15 @@ import Data.Proxy (Proxy (..)) import Data.String (IsString (..)) import qualified Data.Text as T +import GHC.OverloadedLabels (IsLabel (..)) import GHC.TypeLits (KnownSymbol, Symbol, symbolVal) import qualified DataFrame.Functions as F import DataFrame.Internal.Column (Columnable) import DataFrame.Internal.Expression ( - BinaryOp (..), + BinUDF (..), Expr (..), - UnaryOp (..), + UnUDF (..), ) import DataFrame.Internal.Nullable ( BaseType, @@ -185,6 +187,23 @@ ) => TExpr cols a col = TExpr (Col (T.pack (symbolVal (Proxy @name)))) + +{- | Use a column name as an @OverloadedLabels@ label: @#age@ is sugar for +@col \@\"age\"@. Enable @OverloadedLabels@ at the use site. + +@ +adults = filterWhere (#age .>=. lit 18) people +@ +-} +instance + ( KnownSymbol name + , a ~ SafeLookup name cols + , Columnable a + , AssertPresent name cols + ) => + IsLabel name (TExpr cols a) + where + fromLabel = col @name {- | Create a literal expression. Valid for any schema since it references no columns. diff --git a/src/DataFrame/Typed/Operations.hs b/src/DataFrame/Typed/Operations.hs --- a/src/DataFrame/Typed/Operations.hs +++ b/src/DataFrame/Typed/Operations.hs @@ -50,6 +50,12 @@ -- * Vertical merge append, + + -- * Set algebra (topos operations) + union, + intersect, + difference, + symmetricDifference, ) where import Data.Proxy (Proxy (..)) @@ -67,6 +73,7 @@ import qualified DataFrame.Operations.Core as D import DataFrame.Operations.Merge () import qualified DataFrame.Operations.Permutation as D +import qualified DataFrame.Operations.SetOps as DS import qualified DataFrame.Operations.Subset as D import qualified DataFrame.Operations.Transformations as D @@ -340,6 +347,32 @@ -- | Vertically merge two DataFrames with the same schema. append :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols append (TDF a) (TDF b) = TDF (a <> b) + +------------------------------------------------------------------------------- +-- Set algebra (topos operations) +-- +-- Each treats a DataFrame as a /set/ of rows and is schema-preserving: +-- the output type equals the input type; only which rows are present changes. +------------------------------------------------------------------------------- + +-- | Rows appearing in either DataFrame, deduplicated (set union). +union :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols +union (TDF a) (TDF b) = TDF (DS.union a b) + +-- | Rows appearing in both DataFrames, deduplicated (set intersection). +intersect :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols +intersect (TDF a) (TDF b) = TDF (DS.intersect a b) + +{- | Rows in the left DataFrame but not the right, deduplicated +(relational @EXCEPT@; the subobject complement). +-} +difference :: TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols +difference (TDF a) (TDF b) = TDF (DS.difference a b) + +-- | Rows in exactly one of the two DataFrames, deduplicated. +symmetricDifference :: + TypedDataFrame cols -> TypedDataFrame cols -> TypedDataFrame cols +symmetricDifference (TDF a) (TDF b) = TDF (DS.symmetricDifference a b) ------------------------------------------------------------------------------- -- Metadata (pass-through)