diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Revision history for dataframe +## 0.3.4.0 +* Fix right join - previously erased some values in the key. +* Change sort API so we can sort on different rows. +* Add meanMaybe and stddevMaybe that work on `Maybe` values. +* More efficient numeric groupby - use radix sort for indices and pre-sort when collecting. + ## 0.3.3.9 * Fix compilation issue for ghc 9.12.* diff --git a/app/Benchmark.hs b/app/Benchmark.hs --- a/app/Benchmark.hs +++ b/app/Benchmark.hs @@ -8,31 +8,17 @@ import qualified DataFrame.Functions as F import System.Random.Stateful +import DataFrame ((|>)) + main :: IO () main = do - let n = 100_000_000 - g <- newIOGenM =<< newStdGen - let range = (0 :: Double, 1 :: Double) - startGeneration <- getCurrentTime - ns <- VU.replicateM n (uniformRM range g) - xs <- VU.replicateM n (uniformRM range g) - ys <- VU.replicateM n (uniformRM range g) - let df = D.fromUnnamedColumns (map D.fromUnboxedVector [ns, xs, ys]) + df <- D.readCsv "../db-benchmark/data/G1_2e6_1e2_0_0.csv" print df - endGeneration <- getCurrentTime - let generationTime = diffUTCTime endGeneration startGeneration - putStrLn $ "Data generation Time: " ++ show generationTime - startCalculation <- getCurrentTime - print $ D.mean (F.col @Double "0") df - print $ D.variance (F.col @Double "1") df - print $ D.correlation "1" "2" df - endCalculation <- getCurrentTime - let calculationTime = diffUTCTime endCalculation startCalculation - putStrLn $ "Calculation Time: " ++ show calculationTime - startFilter <- getCurrentTime - print $ D.filter (F.col @Double "0") (> 0.971) df D.|> D.take 10 - endFilter <- getCurrentTime - let filterTime = diffUTCTime endFilter startFilter - putStrLn $ "Filter Time: " ++ show filterTime - let totalTime = diffUTCTime endFilter startGeneration - putStrLn $ "Total Time: " ++ show totalTime + start <- getCurrentTime + print $ + df + |> D.groupBy ["id1"] + |> D.aggregate [F.sum (F.col @Int "v1") `F.as` "v1_sum"] + end <- getCurrentTime + let computeTime = diffUTCTime end start + putStrLn $ "Compute Time: " ++ show computeTime diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 0.3.3.9 +version: 0.3.4.0 synopsis: A fast, safe, and intuitive DataFrame library. diff --git a/src/DataFrame/Display/Web/Plot.hs b/src/DataFrame/Display/Web/Plot.hs --- a/src/DataFrame/Display/Web/Plot.hs +++ b/src/DataFrame/Display/Web/Plot.hs @@ -29,6 +29,7 @@ import DataFrame.Internal.Expression import DataFrame.Operations.Core import qualified DataFrame.Operations.Subset as D +import Numeric (showFFloat) import System.Directory import System.Info import System.Process ( @@ -40,7 +41,6 @@ std_out, waitForProcess, ) -import Text.Printf newtype HtmlPlot = HtmlPlot T.Text deriving (Show) @@ -73,10 +73,6 @@ , plotFile = Nothing } -chartJsScript :: T.Text -chartJsScript = - "\n" - generateChartId :: IO T.Text generateChartId = do gen <- newStdGen @@ -96,11 +92,9 @@ , "px;height:" , T.pack (show height) , "px\">\n" - , "\n" + , "\n" , "\n" ] @@ -113,13 +107,17 @@ chartId <- generateChartId let values = extractNumericColumn colName df (minVal, maxVal) = if null values then (0, 1) else (minimum values, maximum values) - numBins = 30 binWidth = (maxVal - minVal) / fromIntegral numBins bins = [minVal + fromIntegral i * binWidth | i <- [0 .. numBins - 1]] counts = calculateHistogram values bins binWidth + precision = max 0 $ ceiling (negate $ logBase 10 binWidth) labels = - T.intercalate "," ["\"" <> T.pack (printf "%.2f" b) <> "\"" | b <- bins] + T.intercalate + "," + [ "\"" <> T.pack (showFFloat (Just precision) b "") <> "\"" + | b <- bins + ] dataPoints = T.intercalate "," [T.pack (show c) | c <- counts] chartTitle = @@ -129,11 +127,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -159,7 +155,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ @@ -192,11 +188,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"scatter\",\n" , " data: {\n" , " datasets: [{\n" @@ -223,7 +217,7 @@ , "\" } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ @@ -287,11 +281,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"scatter\",\n" , " data: {\n" , " datasets: [\n" @@ -311,7 +303,7 @@ , "\" } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ @@ -363,11 +355,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"line\",\n" , " data: {\n" , " labels: [" @@ -387,7 +377,7 @@ , "\" } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ @@ -419,11 +409,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -447,7 +435,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ HtmlPlot $ @@ -465,11 +453,8 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" , "\", {\n" , " type: \"bar\",\n" , " data: {\n" @@ -494,7 +479,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ HtmlPlot $ @@ -519,11 +504,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"pie\",\n" , " data: {\n" , " labels: [" @@ -543,7 +526,7 @@ , chartTitle , "\" }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ HtmlPlot $ @@ -565,11 +548,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"pie\",\n" , " data: {\n" , " labels: [" @@ -589,7 +570,7 @@ , chartTitle , "\" }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ HtmlPlot $ @@ -658,11 +639,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -681,7 +660,7 @@ , " yAxes: [{ stacked: true, ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ @@ -712,11 +691,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -740,7 +717,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ @@ -773,11 +750,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -803,7 +778,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ HtmlPlot $ @@ -827,11 +802,9 @@ jsCode = T.concat - [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" - , "var ctx = document.getElementById('" + [ "setTimeout(function() { new Chart(\"" , chartId - , "').getContext('2d');\n" - , "new Chart(ctx , {\n" + , "\", {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -855,7 +828,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});});" + , "})}, 100);" ] return $ HtmlPlot $ diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs --- a/src/DataFrame/Functions.hs +++ b/src/DataFrame/Functions.hs @@ -34,14 +34,17 @@ import Control.Exception (throw) import Control.Monad +import Control.Monad.IO.Class import qualified Data.Char as Char import Data.Containers.ListUtils import Data.Function +import Data.Functor import qualified Data.List as L import qualified Data.Map as M -import Data.Maybe (fromMaybe, listToMaybe) +import Data.Maybe (catMaybes, fromMaybe, isJust, listToMaybe) import qualified Data.Set as S import qualified Data.Text as T +import qualified Data.Text.IO as T import Data.Time import Data.Type.Equality import qualified Data.Vector as V @@ -49,12 +52,13 @@ import qualified Data.Vector.Unboxed as VU import qualified DataFrame.Operations.Core as D import qualified DataFrame.Operations.Transformations as D -import Debug.Trace (trace, traceShow) +import Debug.Trace (trace) import Language.Haskell.TH import qualified Language.Haskell.TH.Syntax as TH import Text.Regex.TDFA import Type.Reflection (typeRep) -import Prelude hiding (maximum, minimum, sum) +import Prelude hiding (maximum, minimum) +import Prelude as P name :: (Show a) => Expr a -> T.Text name (Col n) = n @@ -166,15 +170,28 @@ sum :: forall a. (Columnable a, Num a, VU.Unbox a) => Expr a -> Expr a sum expr = AggNumericVector expr "sum" VG.sum +sumMaybe :: forall a. (Columnable a, Num a) => Expr (Maybe a) -> Expr a +sumMaybe expr = AggVector expr "sumMaybe" (P.sum . catMaybes . V.toList) + mean :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double mean expr = AggNumericVector expr "mean" mean' +meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double +meanMaybe expr = AggVector expr "meanMaybe" (mean' . optionalToDoubleVector) + variance :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double variance expr = AggNumericVector expr "variance" variance' median :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double median expr = AggNumericVector expr "median" median' +optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double +optionalToDoubleVector = + VU.fromList + . V.foldl' + (\acc e -> if isJust e then realToFrac (fromMaybe 0 e) : acc else acc) + [] + percentile :: Int -> Expr Double -> Expr Double percentile n expr = AggNumericVector @@ -185,6 +202,9 @@ stddev :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double stddev expr = AggNumericVector expr "stddev" (sqrt . variance') +stddevMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double +stddevMaybe expr = AggVector expr "stddevMaybe" (sqrt . variance' . optionalToDoubleVector) + zScore :: Expr Double -> Expr Double zScore c = (c - mean c) / stddev c @@ -770,7 +790,10 @@ maybeType <- lookupTypeName t case maybeType of Just name -> return (ConT name) - Nothing -> fail $ "Unsupported type: " ++ t + Nothing -> + if take 1 t == "[" + then typeFromString [dropFirstAndLast t] <&> AppT ListT + else fail $ "Unsupported type: " ++ t typeFromString [tycon, t1] = do outer <- typeFromString [tycon] inner <- typeFromString [t1] @@ -782,6 +805,9 @@ return (AppT (AppT outer lhs) rhs) typeFromString s = fail $ "Unsupported types: " ++ unwords s +dropFirstAndLast :: [a] -> [a] +dropFirstAndLast = reverse . drop 1 . reverse . drop 1 + declareColumns :: DataFrame -> DecsQ declareColumns df = let @@ -791,7 +817,7 @@ in fmap concat $ forM specs $ \(raw, nm, tyStr) -> do ty <- typeFromString (words tyStr) - traceShow (nm <> " :: Expr " <> T.pack tyStr) (pure ()) + liftIO $ T.putStrLn (nm <> " :: Expr " <> T.pack tyStr) let n = mkName (T.unpack nm) sig <- sigD n [t|Expr $(pure ty)|] val <- valD (varP n) (normalB [|col $(TH.lift raw)|]) [] diff --git a/src/DataFrame/Internal/Expression.hs b/src/DataFrame/Internal/Expression.hs --- a/src/DataFrame/Internal/Expression.hs +++ b/src/DataFrame/Internal/Expression.hs @@ -421,21 +421,58 @@ = UnAggregated Column | Aggregated (TypedColumn a) -mkUnaggregatedColumn :: - forall v a. - (VG.Vector v a, Columnable a) => - v a -> VU.Vector Int -> VU.Vector Int -> V.Vector (v a) -mkUnaggregatedColumn col os indices = - V.generate - (VU.length os - 1) - ( \i -> - VG.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - ( \j -> - col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i))) - ) - ) +mkUnaggregatedColumnBoxed :: + forall a. + (Columnable a) => + V.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (V.Vector a) +mkUnaggregatedColumnBoxed col os indices = + let + sorted = V.unsafeBackpermute col (V.convert indices) + n i = os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i) + start i = os `VG.unsafeIndex` i + in + V.generate + (VU.length os - 1) + ( \i -> + V.unsafeSlice (start i) (n i) sorted + ) +mkUnaggregatedColumnUnboxed :: + forall a. + (Columnable a, VU.Unbox a) => + VU.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (VU.Vector a) +mkUnaggregatedColumnUnboxed col os indices = + let + sorted = VU.unsafeBackpermute col indices + n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i) + start i = os `VG.unsafeIndex` i + in + V.generate + (VU.length os - 1) + ( \i -> + VU.unsafeSlice (start i) (n i) sorted + ) + +mkAggregatedColumnUnboxed :: + forall a b. + (Columnable a, VU.Unbox a, Columnable b, VU.Unbox b) => + VU.Vector a -> + VU.Vector Int -> + VU.Vector Int -> + (VU.Vector a -> b) -> + VU.Vector b +mkAggregatedColumnUnboxed col os indices f = + let + sorted = VU.unsafeBackpermute col indices + n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i) + start i = os `VG.unsafeIndex` i + in + VU.generate + (VU.length os - 1) + ( \i -> + f (VU.unsafeSlice (start i) (n i) sorted) + ) + nestedTypeException :: forall a b. (Typeable a, Typeable b) => String -> DataFrameException nestedTypeException expression = case typeRep @a of @@ -470,9 +507,10 @@ V.replicate (VG.length (offsets gdf) - 1) value interpretAggregation gdf@(Grouped df names indices os) (Col name) = case getColumn name df of Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df) - Just (BoxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices - Just (OptionalColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices - Just (UnboxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices + Just (BoxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumnBoxed col os indices + Just (OptionalColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumnBoxed col os indices + Just (UnboxedColumn col) -> + Right $ UnAggregated $ fromVector $ mkUnaggregatedColumnUnboxed col os indices interpretAggregation gdf expression@(UnaryOp _ (f :: c -> d) expr) = case interpretAggregation @c gdf expr of Left (TypeMismatchException context) -> @@ -738,6 +776,23 @@ } ) (Left e) -> Left e +interpretAggregation gdf@(Grouped df names indices os) expression@(AggNumericVector (Col name) op (f :: VU.Vector b -> c)) = + case getColumn name df of + -- TODO(mchavinda): Fix the compedium of type errors here + -- This is mostly done help with the benchmarking. + Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df) + Just (BoxedColumn col) -> error "Type mismatch." + Just (OptionalColumn col) -> error "Type mismatch." + Just (UnboxedColumn (col :: VU.Vector d)) -> case testEquality (typeRep @b) (typeRep @d) of + Just Refl -> case testEquality (typeRep @c) (typeRep @a) of + Just Refl -> + Right $ + Aggregated $ + TColumn $ + fromUnboxedVector $ + mkAggregatedColumnUnboxed col os indices f + Nothing -> error "Type mismatch" + Nothing -> error "Type mismatch" interpretAggregation gdf@(Grouped df names indices os) expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) = case interpretAggregation @b gdf expr of (Left (TypeMismatchException context)) -> diff --git a/src/DataFrame/Internal/Row.hs b/src/DataFrame/Internal/Row.hs --- a/src/DataFrame/Internal/Row.hs +++ b/src/DataFrame/Internal/Row.hs @@ -11,7 +11,6 @@ import qualified Data.List as L import qualified Data.Map as M -import qualified Data.Set as S import qualified Data.Text as T import qualified Data.Vector as V import qualified Data.Vector.Algorithms.Merge as VA @@ -122,10 +121,9 @@ toRowList :: DataFrame -> [Row] toRowList df = let - nameSet = - S.fromList (map fst (L.sortBy (compare `on` snd) $ M.toList (columnIndices df))) + names = map fst (L.sortBy (compare `on` snd) $ M.toList (columnIndices df)) in - map (mkRowRep df nameSet) [0 .. (fst (dataframeDimensions df) - 1)] + map (mkRowRep df names) [0 .. (fst (dataframeDimensions df) - 1)] {- | Converts the dataframe to a vector of rows with only the specified columns. @@ -149,11 +147,7 @@ Vector of empty rows (one per dataframe row) -} toRowVector :: [T.Text] -> DataFrame -> V.Vector Row -toRowVector names df = - let - nameSet = S.fromList names - in - V.generate (fst (dataframeDimensions df)) (mkRowRep df nameSet) +toRowVector names df = V.generate (fst (dataframeDimensions df)) (mkRowRep df names) mkRowFromArgs :: [T.Text] -> DataFrame -> Int -> Row mkRowFromArgs names df i = V.map get (V.fromList names) @@ -169,11 +163,14 @@ Just (UnboxedColumn column) -> toAny (column VU.! i) Just (OptionalColumn column) -> toAny (column V.! i) -mkRowRep :: DataFrame -> S.Set T.Text -> Int -> Row -mkRowRep df names i = V.generate (S.size names) (\index -> get (names' V.! index)) +-- This function will return the items in the order that is specified +-- by the user. For example, if the dataframe consists of the columns +-- "Age", "Pclass", "Name", and the user asks for ["Name", "Age"], +-- this will order the values in the order ["Mr Smith", 50] +mkRowRep :: DataFrame -> [T.Text] -> Int -> Row +mkRowRep df names i = V.generate (L.length names) (\index -> get (names' V.! index)) where - inOrderIndexes = map fst $ L.sortBy (compare `on` snd) $ M.toList (columnIndices df) - names' = V.fromList [n | n <- inOrderIndexes, S.member n names] + names' = V.fromList names throwError name = error $ "Column " @@ -194,9 +191,16 @@ Nothing -> throw $ ColumnNotFoundException name "mkRowRep" (M.keys $ columnIndices df) -sortedIndexes' :: Bool -> V.Vector Row -> VU.Vector Int -sortedIndexes' asc rows = runST $ do +sortedIndexes' :: [Bool] -> V.Vector Row -> VU.Vector Int +sortedIndexes' flipCompare rows = runST $ do withIndexes <- VG.thaw (V.indexed rows) - VA.sortBy ((if asc then compare else flip compare) `on` snd) withIndexes + VA.sortBy (produceOrderingFromRow flipCompare `on` snd) withIndexes sorted <- VG.unsafeFreeze withIndexes return $ VU.generate (VG.length rows) (\i -> fst (sorted VG.! i)) + +produceOrderingFromRow :: [Bool] -> Row -> Row -> Ordering +produceOrderingFromRow mustFlips v1 v2 = V.foldr (<>) mempty vZipped + where + vFlip = V.fromList mustFlips + vZipped = + V.zipWith3 (\b e1 e2 -> if b then compare e1 e2 else compare e2 e1) vFlip v1 v2 diff --git a/src/DataFrame/Operations/Aggregation.hs b/src/DataFrame/Operations/Aggregation.hs --- a/src/DataFrame/Operations/Aggregation.hs +++ b/src/DataFrame/Operations/Aggregation.hs @@ -12,7 +12,7 @@ import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Vector as V -import qualified Data.Vector.Algorithms.Merge as VA +import qualified Data.Vector.Algorithms.Radix as VA import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU @@ -23,17 +23,15 @@ import DataFrame.Errors import DataFrame.Internal.Column ( Column (..), - Columnable, TypedColumn (..), atIndicesStable, - getIndices, - getIndicesUnboxed, ) import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.Expression +import DataFrame.Internal.Types import DataFrame.Operations.Core import DataFrame.Operations.Subset -import Type.Reflection (typeOf, typeRep) +import Type.Reflection (typeRep) {- | O(k * n) groups the dataframe by the given rows aggregating the remaining rows into vector that should be reduced later. @@ -57,11 +55,15 @@ (VU.fromList (reverse (changingPoints valueIndices))) where indicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` names) (columnIndices df) - rowRepresentations = VU.generate (fst (dimensions df)) (mkRowRep indicesToGroup df) + rowRepresentations = computeRowHashes indicesToGroup df valueIndices = runST $ do withIndexes <- VG.thaw $ VG.indexed rowRepresentations - VA.sortBy (\(a, b) (a', b') -> compare b b') withIndexes + VA.sortBy + (VA.passes @Int 0) + (VA.size @Int 0) + (\p e -> VA.radix 0 (snd e)) + withIndexes VG.unsafeFreeze withIndexes changingPoints :: (Eq a, VU.Unbox a) => VU.Vector (Int, a) -> [Int] @@ -72,43 +74,37 @@ | currentVal == newVal = (offsets, currentVal) | otherwise = (index : offsets, newVal) -mkRowRep :: [Int] -> DataFrame -> Int -> Int -mkRowRep groupColumnIndices df i = case h of - [x] -> x - xs -> hash h +computeRowHashes :: [Int] -> DataFrame -> VU.Vector Int +computeRowHashes indices df = + L.foldl' combineCol initialHashes selectedCols where - h = map mkHash groupColumnIndices - getHashedElem :: Column -> Int -> Int - getHashedElem (BoxedColumn (c :: V.Vector a)) j = hash' @a (c V.! j) - getHashedElem (UnboxedColumn (c :: VU.Vector a)) j = hash' @a (c VU.! j) - getHashedElem (OptionalColumn (c :: V.Vector a)) j = hash' @a (c V.! j) - mkHash j = getHashedElem ((V.!) (columns df) j) i + n = fst (dimensions df) + initialHashes = VU.replicate n 0 -{- | This hash function returns the hash when given a non numeric type but -the value when given a numeric. --} -hash' :: (Columnable a) => a -> Int -hash' value = case testEquality (typeOf value) (typeRep @Double) of - Just Refl -> round $ value * 1000 - Nothing -> case testEquality (typeOf value) (typeRep @Int) of - Just Refl -> value - Nothing -> case testEquality (typeOf value) (typeRep @T.Text) of - Just Refl -> hash value - Nothing -> hash (show value) + selectedCols = map (columns df V.!) indices -mkGroupedColumns :: - VU.Vector Int -> DataFrame -> DataFrame -> T.Text -> DataFrame -mkGroupedColumns indices df acc name = - case (V.!) (columns df) (columnIndices df M.! name) of - BoxedColumn column -> - let vs = indices `getIndices` column - in insertVector name vs acc - OptionalColumn column -> - let vs = indices `getIndices` column - in insertVector name vs acc - UnboxedColumn column -> - let vs = indices `getIndicesUnboxed` column - in insertUnboxedVector name vs acc + combineCol :: VU.Vector Int -> Column -> VU.Vector Int + combineCol acc col = case col of + UnboxedColumn (v :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of + Just Refl -> VU.zipWith hashWithSalt acc v + Nothing -> case testEquality (typeRep @a) (typeRep @Double) of + Just Refl -> VU.zipWith (\h d -> hashWithSalt h (doubleToInt d)) acc v + Nothing -> case sIntegral @a of + STrue -> VU.zipWith (\h d -> hashWithSalt h (fromIntegral @a @Int d)) acc v + SFalse -> case sFloating @a of + STrue -> VU.zipWith (\h d -> hashWithSalt h ((doubleToInt . realToFrac) d)) acc v + SFalse -> VU.zipWith (\h d -> hashWithSalt h (hash (show d))) acc v + BoxedColumn (v :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> VG.convert (V.zipWith hashWithSalt (VG.convert acc) v) + Nothing -> + VG.convert + (V.zipWith (\h d -> hashWithSalt h (hash (show d))) (VG.convert acc) v) + OptionalColumn v -> + VG.convert + (V.zipWith (\h d -> hashWithSalt h (hash (show d))) (VG.convert acc) v) + + doubleToInt :: Double -> Int + doubleToInt = floor {- | Aggregate a grouped dataframe using the expressions given. All ungrouped columns will be dropped. diff --git a/src/DataFrame/Operations/Join.hs b/src/DataFrame/Operations/Join.hs --- a/src/DataFrame/Operations/Join.hs +++ b/src/DataFrame/Operations/Join.hs @@ -71,8 +71,7 @@ [c | (k, c) <- M.toList (D.columnIndices right), k `elem` cs] rightRowRepresentations :: VU.Vector Int - rightRowRepresentations = - VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) + rightRowRepresentations = D.computeRowHashes rightIndicesToGroup right -- Build the Hash Map: Int -> Vector of Indices -- We use ifoldr to efficiently insert (index, key) without intermediate allocations. @@ -90,8 +89,7 @@ [c | (k, c) <- M.toList (D.columnIndices left), k `elem` cs] leftRowRepresentations :: VU.Vector Int - leftRowRepresentations = - VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) + leftRowRepresentations = D.computeRowHashes leftIndicesToGroup left -- Perform the Join (leftIndexChunks, rightIndexChunks) = @@ -173,9 +171,9 @@ leftJoin cs right left = let leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) - leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) + leftRowRepresentations = D.computeRowHashes leftIndicesToGroup left rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right) - rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) + rightRowRepresentations = D.computeRowHashes rightIndicesToGroup right rightKeyCountsAndIndices = VU.foldr (\(i, v) acc -> M.insertWith (++) v [i] acc) @@ -246,66 +244,14 @@ -} rightJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame -rightJoin cs right left = - let - leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) - leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) - leftKeyCountsAndIndicesVec = - M.map VU.fromList $ - VU.foldr - (\(i, v) acc -> M.insertWith (++) v [i] acc) - M.empty - (VU.indexed leftRowRepresentations) - rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right) - rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) - rightRowCount = fst (D.dimensions right) - pairs = - [ (maybeLeft, j) - | j <- [0 .. rightRowCount - 1] - , maybeLeft <- - case M.lookup (rightRowRepresentations VU.! j) leftKeyCountsAndIndicesVec of - Nothing -> [Nothing] - Just lVec -> map Just (VU.toList lVec) - ] - expandedLeftIndicies = VB.fromList (map fst pairs) - expandedRightIndicies = VU.fromList (map snd pairs) - expandedLeft = - left - { columns = VB.map (D.atIndicesWithNulls expandedLeftIndicies) (D.columns left) - , dataframeDimensions = - (VB.length expandedLeftIndicies, snd (D.dataframeDimensions left)) - } - expandedRight = - right - { columns = VB.map (D.atIndicesStable expandedRightIndicies) (D.columns right) - , dataframeDimensions = - (VU.length expandedRightIndicies, snd (D.dataframeDimensions right)) - } - leftColumns = D.columnNames left - rightColumns = D.columnNames right - initDf = expandedLeft - insertIfPresent _ Nothing df = df - insertIfPresent name (Just c) df = D.insertColumn name c df - in - D.fold - ( \name df -> - if name `elem` cs - then df - else - ( if name `elem` leftColumns - then insertIfPresent ("Right_" <> name) (D.getColumn name expandedRight) df - else insertIfPresent name (D.getColumn name expandedRight) df - ) - ) - rightColumns - initDf +rightJoin cs left right = leftJoin cs right left fullOuterJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame fullOuterJoin cs right left = let leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) - leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) + leftRowRepresentations = D.computeRowHashes leftIndicesToGroup left leftKeyCountsAndIndices = VU.foldr (\(i, v) acc -> M.insertWith (++) v [i] acc) @@ -313,7 +259,7 @@ (VU.indexed leftRowRepresentations) leftKeyCountsAndIndicesVec = M.map VU.fromList leftKeyCountsAndIndices rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right) - rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) + rightRowRepresentations = D.computeRowHashes rightIndicesToGroup right rightKeyCountsAndIndices = VU.foldr (\(i, v) acc -> M.insertWith (++) v [i] acc) diff --git a/src/DataFrame/Operations/Permutation.hs b/src/DataFrame/Operations/Permutation.hs --- a/src/DataFrame/Operations/Permutation.hs +++ b/src/DataFrame/Operations/Permutation.hs @@ -16,18 +16,28 @@ import System.Random -- | Sort order taken as a parameter by the 'sortBy' function. -data SortOrder = Ascending | Descending deriving (Eq) +data SortOrder + = Asc T.Text + | Desc T.Text + deriving (Eq) +getSortColumnName :: SortOrder -> T.Text +getSortColumnName (Asc n) = n +getSortColumnName (Desc n) = n + +mustFlipCompare :: SortOrder -> Bool +mustFlipCompare (Asc _) = True +mustFlipCompare (Desc _) = False + {- | O(k log n) Sorts the dataframe by a given row. > sortBy Ascending ["Age"] df -} sortBy :: - SortOrder -> - [T.Text] -> + [SortOrder] -> DataFrame -> DataFrame -sortBy order names df +sortBy sortOrds df | any (`notElem` columnNames df) names = throw $ ColumnNotFoundException @@ -36,9 +46,12 @@ (columnNames df) | otherwise = let - indexes = sortedIndexes' (order == Ascending) (toRowVector names df) + indexes = sortedIndexes' mustFlips (toRowVector names df) in df{columns = V.map (atIndicesStable indexes) (columns df)} + where + names = map getSortColumnName sortOrds + mustFlips = map mustFlipCompare sortOrds shuffle :: (RandomGen g) => diff --git a/src/DataFrame/Operations/Statistics.hs b/src/DataFrame/Operations/Statistics.hs --- a/src/DataFrame/Operations/Statistics.hs +++ b/src/DataFrame/Operations/Statistics.hs @@ -26,6 +26,7 @@ import DataFrame.Internal.DataFrame ( DataFrame (..), columnAsUnboxedVector, + columnAsVector, empty, getColumn, ) @@ -97,6 +98,15 @@ Left e -> throw e Right xs -> mean' xs +meanMaybe :: + forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double +meanMaybe (Col name) df = (mean' . optionalToDoubleVector) (columnAsVector @(Maybe a) name df) +meanMaybe expr df = case interpret @(Maybe a) df expr of + Left e -> throw e + Right (TColumn col) -> case toVector @(Maybe a) col of + Left e -> throw e + Right xs -> (mean' . optionalToDoubleVector) xs + -- | Calculates the median of a given column as a standalone value. median :: forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double @@ -178,6 +188,13 @@ ColumnNotFoundException name "applyStatistic" (M.keys $ columnIndices df) _ -> Nothing {-# INLINE _getColumnAsDouble #-} + +optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double +optionalToDoubleVector = + VU.fromList + . V.foldl' + (\acc e -> if isJust e then realToFrac (fromMaybe 0 e) : acc else acc) + [] -- | Calculates the sum of a given column as a standalone value. sum :: diff --git a/tests/Operations/Aggregations.hs b/tests/Operations/Aggregations.hs --- a/tests/Operations/Aggregations.hs +++ b/tests/Operations/Aggregations.hs @@ -37,6 +37,7 @@ ( testData & D.groupBy ["test1"] & D.aggregate [F.count (F.col @Int "test2") `F.as` "test2"] + & D.sortBy [D.Asc "test1"] ) ) @@ -53,6 +54,7 @@ ( testData & D.groupBy ["test1"] & D.aggregate [F.mean (F.col @Int "test2") `F.as` "test2"] + & D.sortBy [D.Asc "test1"] ) ) @@ -71,6 +73,7 @@ & D.aggregate [ F.mean (F.lift (fromIntegral @Int @Double) (F.col @Int "test2")) `F.as` "test2" ] + & D.sortBy [D.Asc "test1"] ) ) @@ -87,6 +90,7 @@ ( testData & D.groupBy ["test1"] & D.aggregate [F.mean (F.col @Int "test2" + F.col @Int "test2") `F.as` "test2"] + & D.sortBy [D.Asc "test1"] ) ) @@ -106,6 +110,7 @@ [ F.maximum (F.lift (fromIntegral @Int @Double) (F.col @Int "test2")) `F.as` "test2" ] + & D.sortBy [D.Asc "test1"] ) ) @@ -123,6 +128,7 @@ & D.groupBy ["test1"] & D.aggregate [F.maximum (F.col @Int "test2" + F.col @Int "test2") `F.as` "test2"] + & D.sortBy [D.Asc "test1"] ) ) diff --git a/tests/Operations/Join.hs b/tests/Operations/Join.hs --- a/tests/Operations/Join.hs +++ b/tests/Operations/Join.hs @@ -33,7 +33,7 @@ , ("B", D.fromList ["B0" :: Text, "B1", "B2"]) ] ) - (D.sortBy D.Ascending ["key"] (innerJoin ["key"] df1 df2)) + (D.sortBy [D.Asc "key"] (innerJoin ["key"] df1 df2)) ) testLeftJoin :: Test @@ -47,7 +47,7 @@ , ("B", D.fromList [Just "B0", Just "B1" :: Maybe Text, Just "B2"]) ] ) - (D.sortBy D.Ascending ["key"] (leftJoin ["key"] df2 df1)) + (D.sortBy [D.Asc "key"] (leftJoin ["key"] df2 df1)) ) testRightJoin :: Test @@ -61,7 +61,7 @@ , ("B", D.fromList ["B0" :: Text, "B1", "B2"]) ] ) - (D.sortBy D.Ascending ["key"] (rightJoin ["key"] df2 df1)) + (D.sortBy [D.Asc "key"] (rightJoin ["key"] df2 df1)) ) staffDf :: D.DataFrame @@ -108,7 +108,7 @@ ) ] ) - (D.sortBy D.Ascending ["Name"] (fullOuterJoin ["Name"] studentDf staffDf)) + (D.sortBy [D.Asc "Name"] (fullOuterJoin ["Name"] studentDf staffDf)) ) tests :: [Test] diff --git a/tests/Operations/Sort.hs b/tests/Operations/Sort.hs --- a/tests/Operations/Sort.hs +++ b/tests/Operations/Sort.hs @@ -23,6 +23,13 @@ testData :: D.DataFrame testData = D.fromNamedColumns values +moreTestData :: D.DataFrame +moreTestData = + D.fromNamedColumns + [ ("test1", DI.fromList $ replicate 10 (0 :: Int) ++ replicate 10 1) + , ("test2", DI.fromList $ [1 :: Int .. 10] ++ [1 .. 10]) + ] + sortByAscendingWAI :: Test sortByAscendingWAI = TestCase @@ -33,7 +40,7 @@ , ("test2", DI.fromList ['a' .. 'z']) ] ) - (D.sortBy D.Ascending ["test1"] testData) + (D.sortBy [D.Asc "test1"] testData) ) sortByDescendingWAI :: Test @@ -46,16 +53,38 @@ , ("test2", DI.fromList $ reverse ['a' .. 'z']) ] ) - (D.sortBy D.Descending ["test1"] testData) + (D.sortBy [D.Desc "test1"] testData) ) +sortByTwoColumns :: Test +sortByTwoColumns = + TestCase + ( assertEqual + "Sorting moreTestData (which is already sorted) is idempotent." + moreTestData + (D.sortBy [D.Asc "test1", D.Asc "test2"] moreTestData) + ) + +sortByOneColumnAscOneColumnDesc :: Test +sortByOneColumnAscOneColumnDesc = + TestCase + ( assertEqual + "Sorting moreTestData by Desc of test2 reverses the order of the second column." + ( D.fromNamedColumns + [ ("test1", DI.fromList $ replicate 10 (0 :: Int) ++ replicate 10 1) + , ("test2", DI.fromList $ [10 :: Int, 9 .. 1] ++ [10, 9 .. 1]) + ] + ) + (D.sortBy [D.Asc "test1", D.Desc "test2"] moreTestData) + ) + sortByColumnDoesNotExist :: Test sortByColumnDoesNotExist = TestCase ( assertExpectException "[Error Case]" (D.columnNotFound "[\"test0\"]" "sortBy" (D.columnNames testData)) - (print $ D.sortBy D.Ascending ["test0"] testData) + (print $ D.sortBy [D.Asc "test0"] testData) ) tests :: [Test] @@ -63,4 +92,6 @@ [ TestLabel "sortByAscendingWAI" sortByAscendingWAI , TestLabel "sortByDescendingWAI" sortByDescendingWAI , TestLabel "sortByColumnDoesNotExist" sortByColumnDoesNotExist + , TestLabel "sortByTwoColumns" sortByTwoColumns + , TestLabel "sortByOneColumnAscOneColumnDesc" sortByOneColumnAscOneColumnDesc ]