dataframe 0.3.3.9 → 0.3.4.0

raw patch · 14 files changed

+307/−248 lines, 14 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• CHANGELOG.md +6/−0
• app/Benchmark.hs +11/−25
• dataframe.cabal +1/−1
• src/DataFrame/Display/Web/Plot.hs +43/−70
• src/DataFrame/Functions.hs +31/−5
• src/DataFrame/Internal/Expression.hs +72/−17
• src/DataFrame/Internal/Row.hs +20/−16
• src/DataFrame/Operations/Aggregation.hs +37/−41
• src/DataFrame/Operations/Join.hs +7/−61
• src/DataFrame/Operations/Permutation.hs +18/−5
• src/DataFrame/Operations/Statistics.hs +17/−0
• tests/Operations/Aggregations.hs +6/−0
• tests/Operations/Join.hs +4/−4
• tests/Operations/Sort.hs +34/−3

@@ -1,5 +1,11 @@ # Revision history for dataframe +## 0.3.4.0+* Fix right join - previously erased some values in the key.+* Change sort API so we can sort on different rows.+* Add meanMaybe and stddevMaybe that work on `Maybe` values.+* More efficient numeric groupby - use radix sort for indices and pre-sort when collecting.+ ## 0.3.3.9 * Fix compilation issue for ghc 9.12.* 

@@ -8,31 +8,17 @@ import qualified DataFrame.Functions as F import System.Random.Stateful +import DataFrame ((|>))+ main :: IO () main = do-    let n = 100_000_000-    g <- newIOGenM =<< newStdGen-    let range = (0 :: Double, 1 :: Double)-    startGeneration <- getCurrentTime-    ns <- VU.replicateM n (uniformRM range g)-    xs <- VU.replicateM n (uniformRM range g)-    ys <- VU.replicateM n (uniformRM range g)-    let df = D.fromUnnamedColumns (map D.fromUnboxedVector [ns, xs, ys])+    df <- D.readCsv "../db-benchmark/data/G1_2e6_1e2_0_0.csv"     print df-    endGeneration <- getCurrentTime-    let generationTime = diffUTCTime endGeneration startGeneration-    putStrLn $ "Data generation Time: " ++ show generationTime-    startCalculation <- getCurrentTime-    print $ D.mean (F.col @Double "0") df-    print $ D.variance (F.col @Double "1") df-    print $ D.correlation "1" "2" df-    endCalculation <- getCurrentTime-    let calculationTime = diffUTCTime endCalculation startCalculation-    putStrLn $ "Calculation Time: " ++ show calculationTime-    startFilter <- getCurrentTime-    print $ D.filter (F.col @Double "0") (> 0.971) df D.|> D.take 10-    endFilter <- getCurrentTime-    let filterTime = diffUTCTime endFilter startFilter-    putStrLn $ "Filter Time: " ++ show filterTime-    let totalTime = diffUTCTime endFilter startGeneration-    putStrLn $ "Total Time: " ++ show totalTime+    start <- getCurrentTime+    print $+        df+            |> D.groupBy ["id1"]+            |> D.aggregate [F.sum (F.col @Int "v1") `F.as` "v1_sum"]+    end <- getCurrentTime+    let computeTime = diffUTCTime end start+    putStrLn $ "Compute Time: " ++ show computeTime

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.3.3.9+version:            0.3.4.0  synopsis: A fast, safe, and intuitive DataFrame library. 

@@ -29,6 +29,7 @@ import DataFrame.Internal.Expression import DataFrame.Operations.Core import qualified DataFrame.Operations.Subset as D+import Numeric (showFFloat) import System.Directory import System.Info import System.Process (@@ -40,7 +41,6 @@     std_out,     waitForProcess,  )-import Text.Printf  newtype HtmlPlot = HtmlPlot T.Text deriving (Show) @@ -73,10 +73,6 @@         , plotFile = Nothing         } -chartJsScript :: T.Text-chartJsScript =-    "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js\"></script>\n"- generateChartId :: IO T.Text generateChartId = do     gen <- newStdGen@@ -96,11 +92,9 @@         , "px;height:"         , T.pack (show height)         , "px\"></canvas>\n"-        , "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js\"></script>\n"+        , "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js\"></script>\n"         , "<script>\n"-        , "setTimeout(() => {"         , content-        , "}, 200);"         , "\n</script>\n"         ] @@ -113,13 +107,17 @@     chartId <- generateChartId     let values = extractNumericColumn colName df         (minVal, maxVal) = if null values then (0, 1) else (minimum values, maximum values)-        numBins = 30         binWidth = (maxVal - minVal) / fromIntegral numBins         bins = [minVal + fromIntegral i * binWidth | i <- [0 .. numBins - 1]]         counts = calculateHistogram values bins binWidth+        precision = max 0 $ ceiling (negate $ logBase 10 binWidth)          labels =-            T.intercalate "," ["\"" <> T.pack (printf "%.2f" b) <> "\"" | b <- bins]+            T.intercalate+                ","+                [ "\"" <> T.pack (showFFloat (Just precision) b "") <> "\""+                | b <- bins+                ]         dataPoints = T.intercalate "," [T.pack (show c) | c <- counts]          chartTitle =@@ -129,11 +127,9 @@          jsCode =             T.concat-                [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                , "var ctx = document.getElementById('"+                [ "setTimeout(function() { new Chart(\""                 , chartId-                , "').getContext('2d');\n"-                , "new Chart(ctx , {\n"+                , "\", {\n"                 , "  type: \"bar\",\n"                 , "  data: {\n"                 , "    labels: ["@@ -159,7 +155,7 @@                 , "      yAxes: [{ ticks: { beginAtZero: true } }]\n"                 , "    }\n"                 , "  }\n"-                , "});});"+                , "})}, 100);"                 ]      return $@@ -192,11 +188,9 @@          jsCode =             T.concat-                [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                , "var ctx = document.getElementById('"+                [ "setTimeout(function() { new Chart(\""                 , chartId-                , "').getContext('2d');\n"-                , "new Chart(ctx , {\n"+                , "\", {\n"                 , "  type: \"scatter\",\n"                 , "  data: {\n"                 , "    datasets: [{\n"@@ -223,7 +217,7 @@                 , "\" } }]\n"                 , "    }\n"                 , "  }\n"-                , "});});"+                , "})}, 100);"                 ]      return $@@ -287,11 +281,9 @@          jsCode =             T.concat-                [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                , "var ctx = document.getElementById('"+                [ "setTimeout(function() { new Chart(\""                 , chartId-                , "').getContext('2d');\n"-                , "new Chart(ctx , {\n"+                , "\", {\n"                 , "  type: \"scatter\",\n"                 , "  data: {\n"                 , "    datasets: [\n"@@ -311,7 +303,7 @@                 , "\" } }]\n"                 , "    }\n"                 , "  }\n"-                , "});});"+                , "})}, 100);"                 ]      return $@@ -363,11 +355,9 @@          jsCode =             T.concat-                [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                , "var ctx = document.getElementById('"+                [ "setTimeout(function() { new Chart(\""                 , chartId-                , "').getContext('2d');\n"-                , "new Chart(ctx , {\n"+                , "\", {\n"                 , "  type: \"line\",\n"                 , "  data: {\n"                 , "    labels: ["@@ -387,7 +377,7 @@                 , "\" } }]\n"                 , "    }\n"                 , "  }\n"-                , "});});"+                , "})}, 100);"                 ]      return $@@ -419,11 +409,9 @@                  jsCode =                     T.concat-                        [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                        , "var ctx = document.getElementById('"+                        [ "setTimeout(function() { new Chart(\""                         , chartId-                        , "').getContext('2d');\n"-                        , "new Chart(ctx , {\n"+                        , "\", {\n"                         , "  type: \"bar\",\n"                         , "  data: {\n"                         , "    labels: ["@@ -447,7 +435,7 @@                         , "      yAxes: [{ ticks: { beginAtZero: true } }]\n"                         , "    }\n"                         , "  }\n"-                        , "});});"+                        , "})}, 100);"                         ]             return $                 HtmlPlot $@@ -465,11 +453,8 @@                  jsCode =                     T.concat-                        [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                        , "var ctx = document.getElementById('"+                        [ "setTimeout(function() { new Chart(\""                         , chartId-                        , "').getContext('2d');\n"-                        , "new Chart(ctx , {\n"                         , "\", {\n"                         , "  type: \"bar\",\n"                         , "  data: {\n"@@ -494,7 +479,7 @@                         , "      yAxes: [{ ticks: { beginAtZero: true } }]\n"                         , "    }\n"                         , "  }\n"-                        , "});});"+                        , "})}, 100);"                         ]             return $                 HtmlPlot $@@ -519,11 +504,9 @@                  jsCode =                     T.concat-                        [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                        , "var ctx = document.getElementById('"+                        [ "setTimeout(function() { new Chart(\""                         , chartId-                        , "').getContext('2d');\n"-                        , "new Chart(ctx , {\n"+                        , "\", {\n"                         , "  type: \"pie\",\n"                         , "  data: {\n"                         , "    labels: ["@@ -543,7 +526,7 @@                         , chartTitle                         , "\" }\n"                         , "  }\n"-                        , "});});"+                        , "})}, 100);"                         ]             return $                 HtmlPlot $@@ -565,11 +548,9 @@                  jsCode =                     T.concat-                        [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                        , "var ctx = document.getElementById('"+                        [ "setTimeout(function() { new Chart(\""                         , chartId-                        , "').getContext('2d');\n"-                        , "new Chart(ctx , {\n"+                        , "\", {\n"                         , "  type: \"pie\",\n"                         , "  data: {\n"                         , "    labels: ["@@ -589,7 +570,7 @@                         , chartTitle                         , "\" }\n"                         , "  }\n"-                        , "});});"+                        , "})}, 100);"                         ]             return $                 HtmlPlot $@@ -658,11 +639,9 @@          jsCode =             T.concat-                [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                , "var ctx = document.getElementById('"+                [ "setTimeout(function() { new Chart(\""                 , chartId-                , "').getContext('2d');\n"-                , "new Chart(ctx , {\n"+                , "\", {\n"                 , "  type: \"bar\",\n"                 , "  data: {\n"                 , "    labels: ["@@ -681,7 +660,7 @@                 , "      yAxes: [{ stacked: true, ticks: { beginAtZero: true } }]\n"                 , "    }\n"                 , "  }\n"-                , "});});"+                , "})}, 100);"                 ]      return $@@ -712,11 +691,9 @@          jsCode =             T.concat-                [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                , "var ctx = document.getElementById('"+                [ "setTimeout(function() { new Chart(\""                 , chartId-                , "').getContext('2d');\n"-                , "new Chart(ctx , {\n"+                , "\", {\n"                 , "  type: \"bar\",\n"                 , "  data: {\n"                 , "    labels: ["@@ -740,7 +717,7 @@                 , "      yAxes: [{ ticks: { beginAtZero: true } }]\n"                 , "    }\n"                 , "  }\n"-                , "});});"+                , "})}, 100);"                 ]      return $@@ -773,11 +750,9 @@                  jsCode =                     T.concat-                        [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                        , "var ctx = document.getElementById('"+                        [ "setTimeout(function() { new Chart(\""                         , chartId-                        , "').getContext('2d');\n"-                        , "new Chart(ctx , {\n"+                        , "\", {\n"                         , "  type: \"bar\",\n"                         , "  data: {\n"                         , "    labels: ["@@ -803,7 +778,7 @@                         , "      yAxes: [{ ticks: { beginAtZero: true } }]\n"                         , "    }\n"                         , "  }\n"-                        , "});});"+                        , "})}, 100);"                         ]             return $                 HtmlPlot $@@ -827,11 +802,9 @@                  jsCode =                     T.concat-                        [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n"-                        , "var ctx = document.getElementById('"+                        [ "setTimeout(function() { new Chart(\""                         , chartId-                        , "').getContext('2d');\n"-                        , "new Chart(ctx , {\n"+                        , "\", {\n"                         , "  type: \"bar\",\n"                         , "  data: {\n"                         , "    labels: ["@@ -855,7 +828,7 @@                         , "      yAxes: [{ ticks: { beginAtZero: true } }]\n"                         , "    }\n"                         , "  }\n"-                        , "});});"+                        , "})}, 100);"                         ]             return $                 HtmlPlot $

@@ -34,14 +34,17 @@  import Control.Exception (throw) import Control.Monad+import Control.Monad.IO.Class import qualified Data.Char as Char import Data.Containers.ListUtils import Data.Function+import Data.Functor import qualified Data.List as L import qualified Data.Map as M-import Data.Maybe (fromMaybe, listToMaybe)+import Data.Maybe (catMaybes, fromMaybe, isJust, listToMaybe) import qualified Data.Set as S import qualified Data.Text as T+import qualified Data.Text.IO as T import Data.Time import Data.Type.Equality import qualified Data.Vector as V@@ -49,12 +52,13 @@ import qualified Data.Vector.Unboxed as VU import qualified DataFrame.Operations.Core as D import qualified DataFrame.Operations.Transformations as D-import Debug.Trace (trace, traceShow)+import Debug.Trace (trace) import Language.Haskell.TH import qualified Language.Haskell.TH.Syntax as TH import Text.Regex.TDFA import Type.Reflection (typeRep)-import Prelude hiding (maximum, minimum, sum)+import Prelude hiding (maximum, minimum)+import Prelude as P  name :: (Show a) => Expr a -> T.Text name (Col n) = n@@ -166,15 +170,28 @@ sum :: forall a. (Columnable a, Num a, VU.Unbox a) => Expr a -> Expr a sum expr = AggNumericVector expr "sum" VG.sum +sumMaybe :: forall a. (Columnable a, Num a) => Expr (Maybe a) -> Expr a+sumMaybe expr = AggVector expr "sumMaybe" (P.sum . catMaybes . V.toList)+ mean :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double mean expr = AggNumericVector expr "mean" mean' +meanMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double+meanMaybe expr = AggVector expr "meanMaybe" (mean' . optionalToDoubleVector)+ variance :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double variance expr = AggNumericVector expr "variance" variance'  median :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double median expr = AggNumericVector expr "median" median' +optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double+optionalToDoubleVector =+    VU.fromList+        . V.foldl'+            (\acc e -> if isJust e then realToFrac (fromMaybe 0 e) : acc else acc)+            []+ percentile :: Int -> Expr Double -> Expr Double percentile n expr =     AggNumericVector@@ -185,6 +202,9 @@ stddev :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double stddev expr = AggNumericVector expr "stddev" (sqrt . variance') +stddevMaybe :: forall a. (Columnable a, Real a) => Expr (Maybe a) -> Expr Double+stddevMaybe expr = AggVector expr "stddevMaybe" (sqrt . variance' . optionalToDoubleVector)+ zScore :: Expr Double -> Expr Double zScore c = (c - mean c) / stddev c @@ -770,7 +790,10 @@     maybeType <- lookupTypeName t     case maybeType of         Just name -> return (ConT name)-        Nothing -> fail $ "Unsupported type: " ++ t+        Nothing ->+            if take 1 t == "["+                then typeFromString [dropFirstAndLast t] <&> AppT ListT+                else fail $ "Unsupported type: " ++ t typeFromString [tycon, t1] = do     outer <- typeFromString [tycon]     inner <- typeFromString [t1]@@ -782,6 +805,9 @@     return (AppT (AppT outer lhs) rhs) typeFromString s = fail $ "Unsupported types: " ++ unwords s +dropFirstAndLast :: [a] -> [a]+dropFirstAndLast = reverse . drop 1 . reverse . drop 1+ declareColumns :: DataFrame -> DecsQ declareColumns df =     let@@ -791,7 +817,7 @@      in         fmap concat $ forM specs $ \(raw, nm, tyStr) -> do             ty <- typeFromString (words tyStr)-            traceShow (nm <> " :: Expr " <> T.pack tyStr) (pure ())+            liftIO $ T.putStrLn (nm <> " :: Expr " <> T.pack tyStr)             let n = mkName (T.unpack nm)             sig <- sigD n [t|Expr $(pure ty)|]             val <- valD (varP n) (normalB [|col $(TH.lift raw)|]) []

@@ -421,21 +421,58 @@     = UnAggregated Column     | Aggregated (TypedColumn a) -mkUnaggregatedColumn ::-    forall v a.-    (VG.Vector v a, Columnable a) =>-    v a -> VU.Vector Int -> VU.Vector Int -> V.Vector (v a)-mkUnaggregatedColumn col os indices =-    V.generate-        (VU.length os - 1)-        ( \i ->-            VG.generate-                (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i))-                ( \j ->-                    col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))-                )-        )+mkUnaggregatedColumnBoxed ::+    forall a.+    (Columnable a) =>+    V.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (V.Vector a)+mkUnaggregatedColumnBoxed col os indices =+    let+        sorted = V.unsafeBackpermute col (V.convert indices)+        n i = os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)+        start i = os `VG.unsafeIndex` i+     in+        V.generate+            (VU.length os - 1)+            ( \i ->+                V.unsafeSlice (start i) (n i) sorted+            ) +mkUnaggregatedColumnUnboxed ::+    forall a.+    (Columnable a, VU.Unbox a) =>+    VU.Vector a -> VU.Vector Int -> VU.Vector Int -> V.Vector (VU.Vector a)+mkUnaggregatedColumnUnboxed col os indices =+    let+        sorted = VU.unsafeBackpermute col indices+        n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i)+        start i = os `VG.unsafeIndex` i+     in+        V.generate+            (VU.length os - 1)+            ( \i ->+                VU.unsafeSlice (start i) (n i) sorted+            )++mkAggregatedColumnUnboxed ::+    forall a b.+    (Columnable a, VU.Unbox a, Columnable b, VU.Unbox b) =>+    VU.Vector a ->+    VU.Vector Int ->+    VU.Vector Int ->+    (VU.Vector a -> b) ->+    VU.Vector b+mkAggregatedColumnUnboxed col os indices f =+    let+        sorted = VU.unsafeBackpermute col indices+        n i = os `VU.unsafeIndex` (i + 1) - (os `VU.unsafeIndex` i)+        start i = os `VG.unsafeIndex` i+     in+        VU.generate+            (VU.length os - 1)+            ( \i ->+                f (VU.unsafeSlice (start i) (n i) sorted)+            )+ nestedTypeException ::     forall a b. (Typeable a, Typeable b) => String -> DataFrameException nestedTypeException expression = case typeRep @a of@@ -470,9 +507,10 @@                     V.replicate (VG.length (offsets gdf) - 1) value interpretAggregation gdf@(Grouped df names indices os) (Col name) = case getColumn name df of     Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df)-    Just (BoxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices-    Just (OptionalColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices-    Just (UnboxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumn col os indices+    Just (BoxedColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumnBoxed col os indices+    Just (OptionalColumn col) -> Right $ UnAggregated $ fromVector $ mkUnaggregatedColumnBoxed col os indices+    Just (UnboxedColumn col) ->+        Right $ UnAggregated $ fromVector $ mkUnaggregatedColumnUnboxed col os indices interpretAggregation gdf expression@(UnaryOp _ (f :: c -> d) expr) =     case interpretAggregation @c gdf expr of         Left (TypeMismatchException context) ->@@ -738,6 +776,23 @@                         }                     )         (Left e) -> Left e+interpretAggregation gdf@(Grouped df names indices os) expression@(AggNumericVector (Col name) op (f :: VU.Vector b -> c)) =+    case getColumn name df of+        -- TODO(mchavinda): Fix the compedium of type errors here+        -- This is mostly done help with the benchmarking.+        Nothing -> Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df)+        Just (BoxedColumn col) -> error "Type mismatch."+        Just (OptionalColumn col) -> error "Type mismatch."+        Just (UnboxedColumn (col :: VU.Vector d)) -> case testEquality (typeRep @b) (typeRep @d) of+            Just Refl -> case testEquality (typeRep @c) (typeRep @a) of+                Just Refl ->+                    Right $+                        Aggregated $+                            TColumn $+                                fromUnboxedVector $+                                    mkAggregatedColumnUnboxed col os indices f+                Nothing -> error "Type mismatch"+            Nothing -> error "Type mismatch" interpretAggregation gdf@(Grouped df names indices os) expression@(AggNumericVector expr op (f :: VU.Vector b -> c)) =     case interpretAggregation @b gdf expr of         (Left (TypeMismatchException context)) ->

@@ -11,7 +11,6 @@  import qualified Data.List as L import qualified Data.Map as M-import qualified Data.Set as S import qualified Data.Text as T import qualified Data.Vector as V import qualified Data.Vector.Algorithms.Merge as VA@@ -122,10 +121,9 @@ toRowList :: DataFrame -> [Row] toRowList df =     let-        nameSet =-            S.fromList (map fst (L.sortBy (compare `on` snd) $ M.toList (columnIndices df)))+        names = map fst (L.sortBy (compare `on` snd) $ M.toList (columnIndices df))      in-        map (mkRowRep df nameSet) [0 .. (fst (dataframeDimensions df) - 1)]+        map (mkRowRep df names) [0 .. (fst (dataframeDimensions df) - 1)]  {- | Converts the dataframe to a vector of rows with only the specified columns. @@ -149,11 +147,7 @@ Vector of empty rows (one per dataframe row) -} toRowVector :: [T.Text] -> DataFrame -> V.Vector Row-toRowVector names df =-    let-        nameSet = S.fromList names-     in-        V.generate (fst (dataframeDimensions df)) (mkRowRep df nameSet)+toRowVector names df = V.generate (fst (dataframeDimensions df)) (mkRowRep df names)  mkRowFromArgs :: [T.Text] -> DataFrame -> Int -> Row mkRowFromArgs names df i = V.map get (V.fromList names)@@ -169,11 +163,14 @@         Just (UnboxedColumn column) -> toAny (column VU.! i)         Just (OptionalColumn column) -> toAny (column V.! i) -mkRowRep :: DataFrame -> S.Set T.Text -> Int -> Row-mkRowRep df names i = V.generate (S.size names) (\index -> get (names' V.! index))+-- This function will return the items in the order that is specified+-- by the user. For example, if the dataframe consists of the columns+-- "Age", "Pclass", "Name", and the user asks for ["Name", "Age"],+-- this will order the values in the order ["Mr Smith", 50]+mkRowRep :: DataFrame -> [T.Text] -> Int -> Row+mkRowRep df names i = V.generate (L.length names) (\index -> get (names' V.! index))   where-    inOrderIndexes = map fst $ L.sortBy (compare `on` snd) $ M.toList (columnIndices df)-    names' = V.fromList [n | n <- inOrderIndexes, S.member n names]+    names' = V.fromList names     throwError name =         error $             "Column "@@ -194,9 +191,16 @@         Nothing ->             throw $ ColumnNotFoundException name "mkRowRep" (M.keys $ columnIndices df) -sortedIndexes' :: Bool -> V.Vector Row -> VU.Vector Int-sortedIndexes' asc rows = runST $ do+sortedIndexes' :: [Bool] -> V.Vector Row -> VU.Vector Int+sortedIndexes' flipCompare rows = runST $ do     withIndexes <- VG.thaw (V.indexed rows)-    VA.sortBy ((if asc then compare else flip compare) `on` snd) withIndexes+    VA.sortBy (produceOrderingFromRow flipCompare `on` snd) withIndexes     sorted <- VG.unsafeFreeze withIndexes     return $ VU.generate (VG.length rows) (\i -> fst (sorted VG.! i))++produceOrderingFromRow :: [Bool] -> Row -> Row -> Ordering+produceOrderingFromRow mustFlips v1 v2 = V.foldr (<>) mempty vZipped+  where+    vFlip = V.fromList mustFlips+    vZipped =+        V.zipWith3 (\b e1 e2 -> if b then compare e1 e2 else compare e2 e1) vFlip v1 v2

@@ -12,7 +12,7 @@ import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Vector as V-import qualified Data.Vector.Algorithms.Merge as VA+import qualified Data.Vector.Algorithms.Radix as VA import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU @@ -23,17 +23,15 @@ import DataFrame.Errors import DataFrame.Internal.Column (     Column (..),-    Columnable,     TypedColumn (..),     atIndicesStable,-    getIndices,-    getIndicesUnboxed,  ) import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.Expression+import DataFrame.Internal.Types import DataFrame.Operations.Core import DataFrame.Operations.Subset-import Type.Reflection (typeOf, typeRep)+import Type.Reflection (typeRep)  {- | O(k * n) groups the dataframe by the given rows aggregating the remaining rows into vector that should be reduced later.@@ -57,11 +55,15 @@             (VU.fromList (reverse (changingPoints valueIndices)))   where     indicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` names) (columnIndices df)-    rowRepresentations = VU.generate (fst (dimensions df)) (mkRowRep indicesToGroup df)+    rowRepresentations = computeRowHashes indicesToGroup df      valueIndices = runST $ do         withIndexes <- VG.thaw $ VG.indexed rowRepresentations-        VA.sortBy (\(a, b) (a', b') -> compare b b') withIndexes+        VA.sortBy+            (VA.passes @Int 0)+            (VA.size @Int 0)+            (\p e -> VA.radix 0 (snd e))+            withIndexes         VG.unsafeFreeze withIndexes  changingPoints :: (Eq a, VU.Unbox a) => VU.Vector (Int, a) -> [Int]@@ -72,43 +74,37 @@         | currentVal == newVal = (offsets, currentVal)         | otherwise = (index : offsets, newVal) -mkRowRep :: [Int] -> DataFrame -> Int -> Int-mkRowRep groupColumnIndices df i = case h of-    [x] -> x-    xs -> hash h+computeRowHashes :: [Int] -> DataFrame -> VU.Vector Int+computeRowHashes indices df =+    L.foldl' combineCol initialHashes selectedCols   where-    h = map mkHash groupColumnIndices-    getHashedElem :: Column -> Int -> Int-    getHashedElem (BoxedColumn (c :: V.Vector a)) j = hash' @a (c V.! j)-    getHashedElem (UnboxedColumn (c :: VU.Vector a)) j = hash' @a (c VU.! j)-    getHashedElem (OptionalColumn (c :: V.Vector a)) j = hash' @a (c V.! j)-    mkHash j = getHashedElem ((V.!) (columns df) j) i+    n = fst (dimensions df)+    initialHashes = VU.replicate n 0 -{- | This hash function returns the hash when given a non numeric type but-the value when given a numeric.--}-hash' :: (Columnable a) => a -> Int-hash' value = case testEquality (typeOf value) (typeRep @Double) of-    Just Refl -> round $ value * 1000-    Nothing -> case testEquality (typeOf value) (typeRep @Int) of-        Just Refl -> value-        Nothing -> case testEquality (typeOf value) (typeRep @T.Text) of-            Just Refl -> hash value-            Nothing -> hash (show value)+    selectedCols = map (columns df V.!) indices -mkGroupedColumns ::-    VU.Vector Int -> DataFrame -> DataFrame -> T.Text -> DataFrame-mkGroupedColumns indices df acc name =-    case (V.!) (columns df) (columnIndices df M.! name) of-        BoxedColumn column ->-            let vs = indices `getIndices` column-             in insertVector name vs acc-        OptionalColumn column ->-            let vs = indices `getIndices` column-             in insertVector name vs acc-        UnboxedColumn column ->-            let vs = indices `getIndicesUnboxed` column-             in insertUnboxedVector name vs acc+    combineCol :: VU.Vector Int -> Column -> VU.Vector Int+    combineCol acc col = case col of+        UnboxedColumn (v :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of+            Just Refl -> VU.zipWith hashWithSalt acc v+            Nothing -> case testEquality (typeRep @a) (typeRep @Double) of+                Just Refl -> VU.zipWith (\h d -> hashWithSalt h (doubleToInt d)) acc v+                Nothing -> case sIntegral @a of+                    STrue -> VU.zipWith (\h d -> hashWithSalt h (fromIntegral @a @Int d)) acc v+                    SFalse -> case sFloating @a of+                        STrue -> VU.zipWith (\h d -> hashWithSalt h ((doubleToInt . realToFrac) d)) acc v+                        SFalse -> VU.zipWith (\h d -> hashWithSalt h (hash (show d))) acc v+        BoxedColumn (v :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of+            Just Refl -> VG.convert (V.zipWith hashWithSalt (VG.convert acc) v)+            Nothing ->+                VG.convert+                    (V.zipWith (\h d -> hashWithSalt h (hash (show d))) (VG.convert acc) v)+        OptionalColumn v ->+            VG.convert+                (V.zipWith (\h d -> hashWithSalt h (hash (show d))) (VG.convert acc) v)++    doubleToInt :: Double -> Int+    doubleToInt = floor  {- | Aggregate a grouped dataframe using the expressions given. All ungrouped columns will be dropped.

@@ -71,8 +71,7 @@             [c | (k, c) <- M.toList (D.columnIndices right), k `elem` cs]          rightRowRepresentations :: VU.Vector Int-        rightRowRepresentations =-            VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right)+        rightRowRepresentations = D.computeRowHashes rightIndicesToGroup right          -- Build the Hash Map: Int -> Vector of Indices         -- We use ifoldr to efficiently insert (index, key) without intermediate allocations.@@ -90,8 +89,7 @@             [c | (k, c) <- M.toList (D.columnIndices left), k `elem` cs]          leftRowRepresentations :: VU.Vector Int-        leftRowRepresentations =-            VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left)+        leftRowRepresentations = D.computeRowHashes leftIndicesToGroup left          -- Perform the Join         (leftIndexChunks, rightIndexChunks) =@@ -173,9 +171,9 @@ leftJoin cs right left =     let         leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left)-        leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left)+        leftRowRepresentations = D.computeRowHashes leftIndicesToGroup left         rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right)-        rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right)+        rightRowRepresentations = D.computeRowHashes rightIndicesToGroup right         rightKeyCountsAndIndices =             VU.foldr                 (\(i, v) acc -> M.insertWith (++) v [i] acc)@@ -246,66 +244,14 @@ -} rightJoin ::     [T.Text] -> DataFrame -> DataFrame -> DataFrame-rightJoin cs right left =-    let-        leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left)-        leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left)-        leftKeyCountsAndIndicesVec =-            M.map VU.fromList $-                VU.foldr-                    (\(i, v) acc -> M.insertWith (++) v [i] acc)-                    M.empty-                    (VU.indexed leftRowRepresentations)-        rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right)-        rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right)-        rightRowCount = fst (D.dimensions right)-        pairs =-            [ (maybeLeft, j)-            | j <- [0 .. rightRowCount - 1]-            , maybeLeft <--                case M.lookup (rightRowRepresentations VU.! j) leftKeyCountsAndIndicesVec of-                    Nothing -> [Nothing]-                    Just lVec -> map Just (VU.toList lVec)-            ]-        expandedLeftIndicies = VB.fromList (map fst pairs)-        expandedRightIndicies = VU.fromList (map snd pairs)-        expandedLeft =-            left-                { columns = VB.map (D.atIndicesWithNulls expandedLeftIndicies) (D.columns left)-                , dataframeDimensions =-                    (VB.length expandedLeftIndicies, snd (D.dataframeDimensions left))-                }-        expandedRight =-            right-                { columns = VB.map (D.atIndicesStable expandedRightIndicies) (D.columns right)-                , dataframeDimensions =-                    (VU.length expandedRightIndicies, snd (D.dataframeDimensions right))-                }-        leftColumns = D.columnNames left-        rightColumns = D.columnNames right-        initDf = expandedLeft-        insertIfPresent _ Nothing df = df-        insertIfPresent name (Just c) df = D.insertColumn name c df-     in-        D.fold-            ( \name df ->-                if name `elem` cs-                    then df-                    else-                        ( if name `elem` leftColumns-                            then insertIfPresent ("Right_" <> name) (D.getColumn name expandedRight) df-                            else insertIfPresent name (D.getColumn name expandedRight) df-                        )-            )-            rightColumns-            initDf+rightJoin cs left right = leftJoin cs right left  fullOuterJoin ::     [T.Text] -> DataFrame -> DataFrame -> DataFrame fullOuterJoin cs right left =     let         leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left)-        leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left)+        leftRowRepresentations = D.computeRowHashes leftIndicesToGroup left         leftKeyCountsAndIndices =             VU.foldr                 (\(i, v) acc -> M.insertWith (++) v [i] acc)@@ -313,7 +259,7 @@                 (VU.indexed leftRowRepresentations)         leftKeyCountsAndIndicesVec = M.map VU.fromList leftKeyCountsAndIndices         rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right)-        rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right)+        rightRowRepresentations = D.computeRowHashes rightIndicesToGroup right         rightKeyCountsAndIndices =             VU.foldr                 (\(i, v) acc -> M.insertWith (++) v [i] acc)

@@ -16,18 +16,28 @@ import System.Random  -- | Sort order taken as a parameter by the 'sortBy' function.-data SortOrder = Ascending | Descending deriving (Eq)+data SortOrder+    = Asc T.Text+    | Desc T.Text+    deriving (Eq) +getSortColumnName :: SortOrder -> T.Text+getSortColumnName (Asc n) = n+getSortColumnName (Desc n) = n++mustFlipCompare :: SortOrder -> Bool+mustFlipCompare (Asc _) = True+mustFlipCompare (Desc _) = False+ {- | O(k log n) Sorts the dataframe by a given row.  > sortBy Ascending ["Age"] df -} sortBy ::-    SortOrder ->-    [T.Text] ->+    [SortOrder] ->     DataFrame ->     DataFrame-sortBy order names df+sortBy sortOrds df     | any (`notElem` columnNames df) names =         throw $             ColumnNotFoundException@@ -36,9 +46,12 @@                 (columnNames df)     | otherwise =         let-            indexes = sortedIndexes' (order == Ascending) (toRowVector names df)+            indexes = sortedIndexes' mustFlips (toRowVector names df)          in             df{columns = V.map (atIndicesStable indexes) (columns df)}+  where+    names = map getSortColumnName sortOrds+    mustFlips = map mustFlipCompare sortOrds  shuffle ::     (RandomGen g) =>

@@ -26,6 +26,7 @@ import DataFrame.Internal.DataFrame (     DataFrame (..),     columnAsUnboxedVector,+    columnAsVector,     empty,     getColumn,  )@@ -97,6 +98,15 @@         Left e -> throw e         Right xs -> mean' xs +meanMaybe ::+    forall a. (Columnable a, Real a) => Expr (Maybe a) -> DataFrame -> Double+meanMaybe (Col name) df = (mean' . optionalToDoubleVector) (columnAsVector @(Maybe a) name df)+meanMaybe expr df = case interpret @(Maybe a) df expr of+    Left e -> throw e+    Right (TColumn col) -> case toVector @(Maybe a) col of+        Left e -> throw e+        Right xs -> (mean' . optionalToDoubleVector) xs+ -- | Calculates the median of a given column as a standalone value. median ::     forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double@@ -178,6 +188,13 @@             ColumnNotFoundException name "applyStatistic" (M.keys $ columnIndices df)     _ -> Nothing {-# INLINE _getColumnAsDouble #-}++optionalToDoubleVector :: (Real a) => V.Vector (Maybe a) -> VU.Vector Double+optionalToDoubleVector =+    VU.fromList+        . V.foldl'+            (\acc e -> if isJust e then realToFrac (fromMaybe 0 e) : acc else acc)+            []  -- | Calculates the sum of a given column as a standalone value. sum ::

@@ -37,6 +37,7 @@             ( testData                 & D.groupBy ["test1"]                 & D.aggregate [F.count (F.col @Int "test2") `F.as` "test2"]+                & D.sortBy [D.Asc "test1"]             )         ) @@ -53,6 +54,7 @@             ( testData                 & D.groupBy ["test1"]                 & D.aggregate [F.mean (F.col @Int "test2") `F.as` "test2"]+                & D.sortBy [D.Asc "test1"]             )         ) @@ -71,6 +73,7 @@                 & D.aggregate                     [ F.mean (F.lift (fromIntegral @Int @Double) (F.col @Int "test2")) `F.as` "test2"                     ]+                & D.sortBy [D.Asc "test1"]             )         ) @@ -87,6 +90,7 @@             ( testData                 & D.groupBy ["test1"]                 & D.aggregate [F.mean (F.col @Int "test2" + F.col @Int "test2") `F.as` "test2"]+                & D.sortBy [D.Asc "test1"]             )         ) @@ -106,6 +110,7 @@                     [ F.maximum (F.lift (fromIntegral @Int @Double) (F.col @Int "test2"))                         `F.as` "test2"                     ]+                & D.sortBy [D.Asc "test1"]             )         ) @@ -123,6 +128,7 @@                 & D.groupBy ["test1"]                 & D.aggregate                     [F.maximum (F.col @Int "test2" + F.col @Int "test2") `F.as` "test2"]+                & D.sortBy [D.Asc "test1"]             )         ) 

@@ -33,7 +33,7 @@                 , ("B", D.fromList ["B0" :: Text, "B1", "B2"])                 ]             )-            (D.sortBy D.Ascending ["key"] (innerJoin ["key"] df1 df2))+            (D.sortBy [D.Asc "key"] (innerJoin ["key"] df1 df2))         )  testLeftJoin :: Test@@ -47,7 +47,7 @@                 , ("B", D.fromList [Just "B0", Just "B1" :: Maybe Text, Just "B2"])                 ]             )-            (D.sortBy D.Ascending ["key"] (leftJoin ["key"] df2 df1))+            (D.sortBy [D.Asc "key"] (leftJoin ["key"] df2 df1))         )  testRightJoin :: Test@@ -61,7 +61,7 @@                 , ("B", D.fromList ["B0" :: Text, "B1", "B2"])                 ]             )-            (D.sortBy D.Ascending ["key"] (rightJoin ["key"] df2 df1))+            (D.sortBy [D.Asc "key"] (rightJoin ["key"] df2 df1))         )  staffDf :: D.DataFrame@@ -108,7 +108,7 @@                     )                 ]             )-            (D.sortBy D.Ascending ["Name"] (fullOuterJoin ["Name"] studentDf staffDf))+            (D.sortBy [D.Asc "Name"] (fullOuterJoin ["Name"] studentDf staffDf))         )  tests :: [Test]

@@ -23,6 +23,13 @@ testData :: D.DataFrame testData = D.fromNamedColumns values +moreTestData :: D.DataFrame+moreTestData =+    D.fromNamedColumns+        [ ("test1", DI.fromList $ replicate 10 (0 :: Int) ++ replicate 10 1)+        , ("test2", DI.fromList $ [1 :: Int .. 10] ++ [1 .. 10])+        ]+ sortByAscendingWAI :: Test sortByAscendingWAI =     TestCase@@ -33,7 +40,7 @@                 , ("test2", DI.fromList ['a' .. 'z'])                 ]             )-            (D.sortBy D.Ascending ["test1"] testData)+            (D.sortBy [D.Asc "test1"] testData)         )  sortByDescendingWAI :: Test@@ -46,16 +53,38 @@                 , ("test2", DI.fromList $ reverse ['a' .. 'z'])                 ]             )-            (D.sortBy D.Descending ["test1"] testData)+            (D.sortBy [D.Desc "test1"] testData)         ) +sortByTwoColumns :: Test+sortByTwoColumns =+    TestCase+        ( assertEqual+            "Sorting moreTestData (which is already sorted) is idempotent."+            moreTestData+            (D.sortBy [D.Asc "test1", D.Asc "test2"] moreTestData)+        )++sortByOneColumnAscOneColumnDesc :: Test+sortByOneColumnAscOneColumnDesc =+    TestCase+        ( assertEqual+            "Sorting moreTestData by Desc of test2 reverses the order of the second column."+            ( D.fromNamedColumns+                [ ("test1", DI.fromList $ replicate 10 (0 :: Int) ++ replicate 10 1)+                , ("test2", DI.fromList $ [10 :: Int, 9 .. 1] ++ [10, 9 .. 1])+                ]+            )+            (D.sortBy [D.Asc "test1", D.Desc "test2"] moreTestData)+        )+ sortByColumnDoesNotExist :: Test sortByColumnDoesNotExist =     TestCase         ( assertExpectException             "[Error Case]"             (D.columnNotFound "[\"test0\"]" "sortBy" (D.columnNames testData))-            (print $ D.sortBy D.Ascending ["test0"] testData)+            (print $ D.sortBy [D.Asc "test0"] testData)         )  tests :: [Test]@@ -63,4 +92,6 @@     [ TestLabel "sortByAscendingWAI" sortByAscendingWAI     , TestLabel "sortByDescendingWAI" sortByDescendingWAI     , TestLabel "sortByColumnDoesNotExist" sortByColumnDoesNotExist+    , TestLabel "sortByTwoColumns" sortByTwoColumns+    , TestLabel "sortByOneColumnAscOneColumnDesc" sortByOneColumnAscOneColumnDesc     ]