diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Revision history for dataframe +## 0.3.0.0 +* Now supports inner joins +* Aggregations are now expressions allowing for more expressive aggregation logic. +* In GHCI, you can now create type-safe bindings for each column and use those in expressions. +* Added pandas and polars benchmarks. +* Performance improvements to `groupBy`. +* Various bug fixes. + +## 0.2.0.2 +* Experimental Apache Parquet support. +* Rename conversion columns (changed from toColumn and toColumn' to fromVector and fromList). +* Rename constructor for dataframe to fromNamedColumns +* Create an error context for error messages so we can change the exceptions as they are thrown. +* Provide safe versions of building block functions that allow us to build good traces. +* Add readthedocs support. + ## 0.2.0.1 * Fix bug with new comparison expressions. gt and geq were actually implemented as lt and leq. * Changes to make library work with ghc 9.10.1 and 9.12.2 diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,72 +1,182 @@ -# DataFrame - -An intuitive, dynamically-typed DataFrame library. +

+ + dataframe logo + +

-A tool for exploratory data analysis. +
+ + hackage Latest Release + + + C/I + +
-## Installing +

+ User guide + | + Discord +

-### CLI -* Install Haskell (ghc + cabal) via [ghcup](https://www.haskell.org/ghcup/install/) selecting all the default options. -* To install dataframe run `cabal update && cabal install dataframe` -* Open a Haskell repl with dataframe loaded by running `cabal repl --build-depends dataframe`. -* Follow along any one of the tutorials below. +# DataFrame -### Jupyter notebook -* Use the Dockerfile in the [ihaskell-dataframe](https://github.com/mchav/ihaskell-dataframe) to build and run an image with dataframe integration. -* For a preview check out the [California Housing](https://github.com/mchav/dataframe/blob/main/docs/California%20Housing.ipynb) notebook. +A fast, safe, and intuitive DataFrame library. -## What is exploratory data analysis? -We provide a primer [here](https://github.com/mchav/dataframe/blob/main/docs/exploratory_data_analysis_primer.md) and show how to do some common analyses. +## Why use this DataFrame library? -## Coming from other dataframe libraries -Familiar with another dataframe library? Get started: -* [Coming from Pandas](https://github.com/mchav/dataframe/blob/main/docs/coming_from_pandas.md) -* [Coming from Polars](https://github.com/mchav/dataframe/blob/main/docs/coming_from_polars.md) -* [Coming from dplyr](https://github.com/mchav/dataframe/blob/main/docs/coming_from_dplyr.md) +* Encourages concise, declarative, and composable data pipelines. +* Static typing makes code easier to reason about and catches many bugs at compile time—before your code ever runs. +* Delivers high performance thanks to Haskell’s optimizing compiler and efficient memory model. +* Designed for interactivity: expressive syntax, helpful error messages, and sensible defaults. ## Example usage -### Code example +### Interactive environment ```haskell -import qualified DataFrame as D +ghci> import qualified DataFrame as D +ghci> import DataFrame ((|>)) +ghci> df <- D.readCsv "./data/housing.csv" +ghci> D.columnInfo df +-------------------------------------------------------------------------------------------------------------------- +index | Column Name | # Non-null Values | # Null Values | # Partially parsed | # Unique Values | Type +------|--------------------|-------------------|---------------|--------------------|-----------------|------------- + Int | Text | Int | Int | Int | Int | Text +------|--------------------|-------------------|---------------|--------------------|-----------------|------------- +0 | total_bedrooms | 20433 | 207 | 0 | 1924 | Maybe Double +1 | ocean_proximity | 20640 | 0 | 0 | 5 | Text +2 | median_house_value | 20640 | 0 | 0 | 3842 | Double +3 | median_income | 20640 | 0 | 0 | 12928 | Double +4 | households | 20640 | 0 | 0 | 1815 | Double +5 | population | 20640 | 0 | 0 | 3888 | Double +6 | total_rooms | 20640 | 0 | 0 | 5926 | Double +7 | housing_median_age | 20640 | 0 | 0 | 52 | Double +8 | latitude | 20640 | 0 | 0 | 862 | Double +9 | longitude | 20640 | 0 | 0 | 844 | Double +ghci> :exposeColumns df +ghci> import qualified DataFrame.Functions as F +ghci> df |> D.groupBy ["ocean_proximity"] |> D.aggregate [(F.mean median_house_value) `F.as` "avg_house_value" ] +-------------------------------------------- +index | ocean_proximity | avg_house_value +------|-----------------|------------------- + Int | Text | Double +------|-----------------|------------------- +0 | <1H OCEAN | 240084.28546409807 +1 | INLAND | 124805.39200122119 +2 | ISLAND | 380440.0 +3 | NEAR BAY | 259212.31179039303 +4 | NEAR OCEAN | 249433.97742663656 +ghci> df |> D.derive "rooms_per_household" (total_rooms / households) |> D.take 10 +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +index | longitude | latitude | housing_median_age | total_rooms | total_bedrooms | population | households | median_income | median_house_value | ocean_proximity | rooms_per_household +------|-----------|----------|--------------------|-------------|----------------|------------|------------|--------------------|--------------------|-----------------|-------------------- + Int | Double | Double | Double | Double | Maybe Double | Double | Double | Double | Double | Text | Double +------|-----------|----------|--------------------|-------------|----------------|------------|------------|--------------------|--------------------|-----------------|-------------------- +0 | -122.23 | 37.88 | 41.0 | 880.0 | Just 129.0 | 322.0 | 126.0 | 8.3252 | 452600.0 | NEAR BAY | 6.984126984126984 +1 | -122.22 | 37.86 | 21.0 | 7099.0 | Just 1106.0 | 2401.0 | 1138.0 | 8.3014 | 358500.0 | NEAR BAY | 6.238137082601054 +2 | -122.24 | 37.85 | 52.0 | 1467.0 | Just 190.0 | 496.0 | 177.0 | 7.2574 | 352100.0 | NEAR BAY | 8.288135593220339 +3 | -122.25 | 37.85 | 52.0 | 1274.0 | Just 235.0 | 558.0 | 219.0 | 5.6431000000000004 | 341300.0 | NEAR BAY | 5.8173515981735155 +4 | -122.25 | 37.85 | 52.0 | 1627.0 | Just 280.0 | 565.0 | 259.0 | 3.8462 | 342200.0 | NEAR BAY | 6.281853281853282 +5 | -122.25 | 37.85 | 52.0 | 919.0 | Just 213.0 | 413.0 | 193.0 | 4.0368 | 269700.0 | NEAR BAY | 4.761658031088083 +6 | -122.25 | 37.84 | 52.0 | 2535.0 | Just 489.0 | 1094.0 | 514.0 | 3.6591 | 299200.0 | NEAR BAY | 4.9319066147859925 +7 | -122.25 | 37.84 | 52.0 | 3104.0 | Just 687.0 | 1157.0 | 647.0 | 3.12 | 241400.0 | NEAR BAY | 4.797527047913447 +8 | -122.26 | 37.84 | 42.0 | 2555.0 | Just 665.0 | 1206.0 | 595.0 | 2.0804 | 226700.0 | NEAR BAY | 4.294117647058823 +9 | -122.25 | 37.84 | 52.0 | 3549.0 | Just 707.0 | 1551.0 | 714.0 | 3.6912000000000003 | 261100.0 | NEAR BAY | 4.970588235294118 +ghci> df |> D.derive "nonsense_feature" (latitude + ocean_proximity) |> D.take 10 -import DataFrame ((|>)) +:14:47: error: [GHC-83865] + • Couldn't match type ‘Text’ with ‘Double’ + Expected: Expr Double + Actual: Expr Text + • In the second argument of ‘(+)’, namely ‘ocean_proximity’ + In the second argument of ‘derive’, namely + ‘(latitude + ocean_proximity)’ + In the second argument of ‘(|>)’, namely + ‘derive "nonsense_feature" (latitude + ocean_proximity)’ +``` +Key features in example: +* Intuitive, SQL-like API to get from data to insights. +* Create type-safe references to columns in a dataframe using `:exponseColumns` +* Type-safe column transformations for faster and safer exploration. +* Fluid, chaining API that makes code easy to reason about. + +### Standalone script example +```haskell +-- Useful Haskell extensions. +{-# LANGUAGE OverloadedStrings #-} -- Allow string literal to be interpreted as any other string type. +{-# LANGUAGE TypeApplications #-} -- Convenience syntax for specifiying the type `sum a b :: Int` vs `sum @Int a b'. + +import qualified DataFrame as D -- import for general functionality. +import qualified DataFrame.Functions as F -- import for column expressions. + +import DataFrame ((|>)) -- import chaining operator with unqualified. + main :: IO () +main = do df <- D.readTsv "./data/chipotle.tsv" - print $ df + let quantity = F.col "quantity" :: D.Expr Int -- A typed reference to a column. + print (df |> D.select ["item_name", "quantity"] |> D.groupBy ["item_name"] - |> D.aggregate (zip (repeat "quantity") [D.Maximum, D.Mean, D.Sum]) - |> D.sortBy D.Descending ["Sum_quantity"] + |> D.aggregate [ (F.sum quantity) `F.as` "sum_quantity" + , (F.mean quantity) `F.as` "mean_quantity" + , (F.maximum quantity) `F.as` "maximum_quantity" + ] + |> D.sortBy D.Descending ["sum_quantity"] + |> D.take 10) + ``` Output: ``` ----------------------------------------------------------------------------------------------------- -index | item_name | Sum_quantity | Mean_quantity | Maximum_quantity -------|---------------------------------------|--------------|--------------------|----------------- - Int | Text | Int | Double | Int -------|---------------------------------------|--------------|--------------------|----------------- -0 | Chips and Fresh Tomato Salsa | 130 | 1.1818181818181819 | 15 -1 | Izze | 22 | 1.1 | 3 -2 | Nantucket Nectar | 31 | 1.1481481481481481 | 3 -3 | Chips and Tomatillo-Green Chili Salsa | 35 | 1.1290322580645162 | 3 -4 | Chicken Bowl | 761 | 1.0482093663911847 | 3 -5 | Side of Chips | 110 | 1.0891089108910892 | 8 -6 | Steak Burrito | 386 | 1.048913043478261 | 3 -7 | Steak Soft Tacos | 56 | 1.018181818181818 | 2 -8 | Chips and Guacamole | 506 | 1.0563674321503131 | 4 -9 | Chicken Crispy Tacos | 50 | 1.0638297872340425 | 2 +------------------------------------------------------------------------------------------ +index | item_name | sum_quantity | mean_quanity | maximum_quanity +------|------------------------------|--------------|--------------------|---------------- + Int | Text | Int | Double | Int +------|------------------------------|--------------|--------------------|---------------- +0 | Chicken Bowl | 761 | 1.0482093663911847 | 3 +1 | Chicken Burrito | 591 | 1.0687160940325497 | 4 +2 | Chips and Guacamole | 506 | 1.0563674321503131 | 4 +3 | Steak Burrito | 386 | 1.048913043478261 | 3 +4 | Canned Soft Drink | 351 | 1.1661129568106312 | 4 +5 | Chips | 230 | 1.0900473933649288 | 3 +6 | Steak Bowl | 221 | 1.04739336492891 | 3 +7 | Bottled Water | 211 | 1.3024691358024691 | 10 +8 | Chips and Fresh Tomato Salsa | 130 | 1.1818181818181819 | 15 +9 | Canned Soda | 126 | 1.2115384615384615 | 4 ``` -Full example in `./app` folder using many of the constructs in the API. +Full example in `./examples` folder using many of the constructs in the API. ### Visual example ![Screencast of usage in GHCI](./static/example.gif) +## Installing + +### Jupyter notebook +* We have a [hosted version of the Jupyter notebook](https://ihaskell-dataframe-crf7g5fvcpahdegz.westus2-01.azurewebsites.net/lab/) on azure sites. +* Use the Dockerfile in the [ihaskell-dataframe](https://github.com/mchav/ihaskell-dataframe) to build and run an image with dataframe integration. +* For a preview check out the [California Housing](https://ihaskell-dataframe-crf7g5fvcpahdegz.westus2-01.azurewebsites.net/lab/tree/California%20Housing.ipynb) notebook. + +### CLI +* Install Haskell (ghc + cabal) via [ghcup](https://www.haskell.org/ghcup/install/) selecting all the default options. +* Install snappy (needed for Parquet support) by running: `sudo apt install libsnappy-dev`. +* To install dataframe run `cabal update && cabal install dataframe` +* Open a Haskell repl with dataframe loaded by running `cabal repl --build-depends dataframe`. +* Follow along any one of the tutorials below. + + +## What is exploratory data analysis? +We provide a primer [here](https://github.com/mchav/dataframe/blob/main/docs/exploratory_data_analysis_primer.md) and show how to do some common analyses. + +## Coming from other dataframe libraries +Familiar with another dataframe library? Get started: +* [Coming from Pandas](https://github.com/mchav/dataframe/blob/main/docs/coming_from_pandas.md) +* [Coming from Polars](https://github.com/mchav/dataframe/blob/main/docs/coming_from_polars.md) +* [Coming from dplyr](https://github.com/mchav/dataframe/blob/main/docs/coming_from_dplyr.md) + ## Supported input formats * CSV * Apache Parquet (still buggy and experimental) @@ -75,6 +185,4 @@ * Apache arrow compatability * Integration with common data formats (currently only supports CSV) * Support windowed plotting (currently only supports ASCII plots) - -## Contributing -* Please first submit an issue and we can discuss there. +* Host the whole library + Jupyter lab on Azure with auth and isolation. diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -1,147 +1,23 @@ -{-# LANGUAGE ExtendedDefaultRules #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE TypeApplications #-} -{-# LANGUAGE TupleSections #-} -module Main where - -import qualified DataFrame as D -import DataFrame (dimensions, (|>)) -import Data.List (delete) -import Data.Maybe (fromMaybe, isJust, isNothing) -import qualified Data.Text as T -import qualified Data.Vector as V -import qualified Data.Vector.Generic as VG -import qualified Data.Vector.Unboxed as VU +-- Useful Haskell extensions. +{-# LANGUAGE OverloadedStrings #-} -- Allow string literal to be interpreted as any other string type. +{-# LANGUAGE TypeApplications #-} -- Convenience syntax for specifiying the type `sum a b :: Int` vs `sum @Int a b'. --- Numbers default to int and double, and strings to text -default (Int, T.Text, Double) +import qualified DataFrame as D -- import for general functionality. +import qualified DataFrame.Functions as F -- import for column expressions. --- Example usage of DataFrame library +import DataFrame ((|>)) -- import chaining operator with unqualified. main :: IO () main = do - putStrLn "Housing" - housing - putStrLn $ replicate 100 '-' - - putStrLn "Chipotle Data" - chipotle - putStrLn $ replicate 100 '-' - - putStrLn "One Billion Row Challenge" - oneBillingRowChallenge - putStrLn $ replicate 100 '-' - - putStrLn "Covid Data" - covid - putStrLn $ replicate 100 '-' - - -mean :: (Fractional a, VG.Vector v a) => v a -> a -mean xs = VG.sum xs / fromIntegral (VG.length xs) - -oneBillingRowChallenge :: IO () -oneBillingRowChallenge = do - parsed <- D.readSeparated ';' D.defaultOptions "./data/measurements.txt" - print $ - parsed - |> D.groupBy ["City"] - |> D.reduceBy (\v -> (VG.minimum v, mean @Double v, VG.maximum v)) "Measurement" - |> D.sortBy D.Ascending ["City"] - -housing :: IO () -housing = do - parsed <- D.readCsv "./data/housing.csv" - - print $ D.columnInfo parsed - - -- Sample. - print $ D.take 5 parsed - - D.plotHistograms D.PlotAll D.VerticalHistogram parsed - -covid :: IO () -covid = do - rawFrame <- D.readCsv "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv" - print $ dimensions rawFrame - print $ D.take 10 rawFrame - - D.plotHistograms D.PlotAll D.VerticalHistogram rawFrame - - -- value of all exports from 2015 - print $ - rawFrame - |> D.filter "Direction" (== "Exports") - |> D.select ["Direction", "Year", "Country", "Value"] - |> D.groupBy ["Direction", "Year", "Country"] - |> D.reduceByAgg D.Sum "Value" - -chipotle :: IO () -chipotle = do - rawFrame <- D.readTsv "./data/chipotle.tsv" - print $ D.dimensions rawFrame - - -- -- Sampling the dataframe - print $ D.take 5 rawFrame - - -- Transform the data from a raw string into - -- respective types (throws error on failure) - let f = - rawFrame - -- Change a specfic order ID - |> D.applyWhere (== 1) "order_id" (+ 2) "quantity" - -- Index based change. - |> D.applyAtIndex 0 (\n -> n - 2) "quantity" - -- Custom parsing: drop dollar sign and parse price as double - |> D.apply (D.readValue @Double . T.drop 1) "item_price" - - -- sample the dataframe. - print $ D.take 10 f - - -- Create a total_price column that is quantity * item_price - let withTotalPrice = D.derive "total_price" (D.lift fromIntegral (D.col @Int "quantity") * D.col @Double"item_price") f - - -- sample a filtered subset of the dataframe - putStrLn "Sample dataframe" - print $ - withTotalPrice - |> D.select ["quantity", "item_name", "item_price", "total_price"] - |> D.filter "total_price" (100.0 <) - |> D.take 10 - - -- Check how many chicken burritos were ordered. - -- There are two ways to checking how many chicken burritos - -- were ordered. - let searchTerm = "Chicken Burrito" :: T.Text - - print $ - f - |> D.select ["item_name", "quantity"] - -- It's more efficient to filter before grouping. - |> D.filter "item_name" (searchTerm ==) - |> D.groupBy ["item_name"] - -- can also be written as: - -- D.aggregate (zip (repeat "quantity") [D.Sum, D.Maximum, D.Mean]) - |> D.aggregate (map ("quantity",) [D.Sum, D.Maximum, D.Mean]) - -- Automatically create a variable called _ - |> D.sortBy D.Descending ["Sum_quantity"] - - -- Similarly, we can aggregate quantities by all rows. - print $ - f + df <- D.readTsv "./data/chipotle.tsv" + let quantity = F.col "quantity" :: D.Expr Int -- A typed reference to a column. + print (df |> D.select ["item_name", "quantity"] |> D.groupBy ["item_name"] - -- Aggregate written more explicitly. - -- We have the full expressiveness of Haskell and we needn't fall - -- use a DSL. - |> D.aggregate [("quantity", D.Maximum), ("quantity", D.Mean), ("quantity", D.Sum)] - |> D.take 10 - - let firstOrder = - withTotalPrice - |> D.filterBy (maybe False (T.isInfixOf "Guacamole")) "choice_description" - |> D.filterBy (("Chicken Bowl" :: T.Text) ==) "item_name" - - print $ D.take 10 firstOrder + |> D.aggregate [ (F.sum quantity) `F.as` "sum_quantity" + , (F.mean quantity) `F.as` "mean_quantity" + , (F.maximum quantity) `F.as` "maximum_quantity" + ] + |> D.sortBy D.Descending ["sum_quantity"] + |> D.take 10) diff --git a/benchmark/Main.hs b/benchmark/Main.hs --- a/benchmark/Main.hs +++ b/benchmark/Main.hs @@ -1,30 +1,60 @@ {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} import qualified DataFrame as D +import qualified DataFrame.Functions as F import qualified Data.Vector.Unboxed as VU +import qualified Data.Vector.Unboxed.Mutable as VUM import Control.Monad (replicateM) import Criterion.Main -import System.Random (randomRIO) +import DataFrame ((|>)) +import Data.Time +import System.Process +import System.Random.Stateful -stats :: Int -> IO () -stats n = do - ns <- VU.replicateM n (randomRIO (-20.0 :: Double, 20.0)) - xs <- VU.replicateM n (randomRIO (-20.0 :: Double, 20.0)) - ys <- VU.replicateM n (randomRIO (-20.0 :: Double, 20.0)) - let df = D.fromNamedColumns [("first", D.UnboxedColumn ns), - ("second", D.UnboxedColumn xs), - ("third", D.UnboxedColumn ys)] - - print $ D.mean "first" df - print $ D.variance "second" df - print $ D.correlation "second" "third" df - print $ D.select ["first"] df D.|> D.take 1 +haskell :: IO () +haskell = do + output <- readProcess "cabal" ["run", "dataframe"] "" + putStrLn output -main = defaultMain [ - bgroup "stats" [ bench "300_000" $ nfIO (stats 100_000) - , bench "3_000_000" $ nfIO (stats 1_000_000) - , bench "30_000_000" $ nfIO (stats 30_000_000) - ] - ] +polars :: IO () +polars = do + output <- readProcess "./benchmark/dataframe_benchmark/bin/python3" ["./benchmark/polars/polars_benchmark.py"] "" + putStrLn output + +pandas :: IO () +pandas = do + output <- readProcess "./benchmark/dataframe_benchmark/bin/python3" ["./benchmark/pandas/pandas_benchmark.py"] "" + putStrLn output + +groupByHaskell :: IO () +groupByHaskell = do + df <- D.readCsv "./data/housing.csv" + print $ df |> D.groupBy ["ocean_proximity"] + |> D.aggregate [ (F.minimum (F.col @Double "median_house_value")) `F.as` "minimum_median_house_value" + , (F.maximum (F.col @Double "median_house_value")) `F.as` "maximum_median_house_value"] + +groupByPolars :: IO () +groupByPolars = do + output <- readProcess "./benchmark/dataframe_benchmark/bin/python3" ["./benchmark/polars/group_by.py"] "" + putStrLn output + +groupByPandas :: IO () +groupByPandas = do + output <- readProcess "./benchmark/dataframe_benchmark/bin/python3" ["./benchmark/pandas/group_by.py"] "" + putStrLn output + +main = do + output <- readProcess "cabal" ["build", "-O2"] "" + putStrLn output + defaultMain [ + bgroup "stats" [ bench "simpleStatsHaskell" $ nfIO haskell + , bench "simpleStatsPandas" $ nfIO pandas + , bench "simpleStatsPolars" $ nfIO polars + , bench "groupByHaskell" $ nfIO groupByHaskell + , bench "groupByPolars" $ nfIO groupByPolars + , bench "groupByPandas" $ nfIO groupByPandas + ] + ] diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,10 +1,10 @@ cabal-version: 2.4 name: dataframe -version: 0.2.0.2 +version: 0.3.0.0 -synopsis: An intuitive, dynamically-typed DataFrame library. +synopsis: A fast, safe, and intuitive DataFrame library. -description: An intuitive, dynamically-typed DataFrame library for exploratory data analysis. +description: A fast, safe, and intuitive DataFrame library for exploratory data analysis. bug-reports: https://github.com/mchav/dataframe/issues license: GPL-3.0-or-later @@ -22,8 +22,10 @@ location: https://github.com/mchav/dataframe library + -- default-extensions: StrictData exposed-modules: DataFrame, - DataFrame.Lazy + DataFrame.Lazy, + DataFrame.Functions other-modules: DataFrame.Internal.Types, DataFrame.Internal.Expression, DataFrame.Internal.Parsing, @@ -34,6 +36,7 @@ DataFrame.Internal.Row, DataFrame.Errors, DataFrame.Operations.Core, + DataFrame.Operations.Join, DataFrame.Operations.Merge, DataFrame.Operations.Subset, DataFrame.Operations.Sorting, @@ -56,6 +59,7 @@ hashable >= 1.2 && <= 1.5.0.0, snappy >= 0.2.0.0 && <= 0.2.0.4, statistics >= 0.16.2.1 && <= 0.16.3.0, + template-haskell >= 2.0 && <= 2.30, text >= 2.0 && <= 2.1.2, time >= 1.12 && <= 1.14, vector ^>= 0.13, @@ -64,6 +68,141 @@ hs-source-dirs: src default-language: Haskell2010 +executable chipotle + main-is: Chipotle.hs + other-modules: DataFrame, + DataFrame.Internal.Types, + DataFrame.Internal.Expression, + DataFrame.Internal.Parsing, + DataFrame.Internal.Column, + DataFrame.Display.Terminal.PrettyPrint, + DataFrame.Display.Terminal.Colours, + DataFrame.Internal.DataFrame, + DataFrame.Internal.Row, + DataFrame.Errors, + DataFrame.Operations.Core, + DataFrame.Operations.Subset, + DataFrame.Operations.Sorting, + DataFrame.Operations.Statistics, + DataFrame.Operations.Transformations, + DataFrame.Operations.Typing, + DataFrame.Operations.Aggregation, + DataFrame.Display.Terminal.Plot, + DataFrame.IO.CSV, + DataFrame.Operations.Join, + DataFrame.Operations.Merge, + DataFrame.IO.Parquet, + DataFrame.Lazy.IO.CSV, + DataFrame.Functions + build-depends: base >= 4.17.2.0 && < 4.22, + array ^>= 0.5, + attoparsec >= 0.12 && <= 0.14.4, + bytestring >= 0.11 && <= 0.12.2.0, + containers >= 0.6.7 && < 0.8, + directory >= 1.3.0.0 && <= 1.3.9.0, + hashable >= 1.2 && <= 1.5.0.0, + snappy >= 0.2.0.0 && <= 0.2.0.4, + statistics >= 0.16.2.1 && <= 0.16.3.0, + template-haskell >= 2.0 && <= 2.30, + text >= 2.0 && <= 2.1.2, + time >= 1.12 && <= 1.14, + vector ^>= 0.13, + vector-algorithms ^>= 0.9, + zstd >= 0.1.2.0 && <= 0.1.3.0 + hs-source-dirs: examples, + src + default-language: Haskell2010 + +executable california_housing + main-is: CaliforniaHousing.hs + other-modules: DataFrame, + DataFrame.Internal.Types, + DataFrame.Internal.Expression, + DataFrame.Internal.Parsing, + DataFrame.Internal.Column, + DataFrame.Display.Terminal.PrettyPrint, + DataFrame.Display.Terminal.Colours, + DataFrame.Internal.DataFrame, + DataFrame.Internal.Row, + DataFrame.Errors, + DataFrame.Operations.Core, + DataFrame.Operations.Subset, + DataFrame.Operations.Sorting, + DataFrame.Operations.Statistics, + DataFrame.Operations.Transformations, + DataFrame.Operations.Typing, + DataFrame.Operations.Aggregation, + DataFrame.Display.Terminal.Plot, + DataFrame.IO.CSV, + DataFrame.Operations.Join, + DataFrame.Operations.Merge, + DataFrame.IO.Parquet, + DataFrame.Lazy.IO.CSV, + DataFrame.Functions + build-depends: base >= 4.17.2.0 && < 4.22, + array ^>= 0.5, + attoparsec >= 0.12 && <= 0.14.4, + bytestring >= 0.11 && <= 0.12.2.0, + containers >= 0.6.7 && < 0.8, + directory >= 1.3.0.0 && <= 1.3.9.0, + hashable >= 1.2 && <= 1.5.0.0, + snappy >= 0.2.0.0 && <= 0.2.0.4, + statistics >= 0.16.2.1 && <= 0.16.3.0, + template-haskell >= 2.0 && <= 2.30, + text >= 2.0 && <= 2.1.2, + time >= 1.12 && <= 1.14, + vector ^>= 0.13, + vector-algorithms ^>= 0.9, + zstd >= 0.1.2.0 && <= 0.1.3.0 + hs-source-dirs: examples, + src + default-language: Haskell2010 + +executable one_billion_row_challenge + main-is: OneBillionRowChallenge.hs + other-modules: DataFrame, + DataFrame.Internal.Types, + DataFrame.Internal.Expression, + DataFrame.Internal.Parsing, + DataFrame.Internal.Column, + DataFrame.Display.Terminal.PrettyPrint, + DataFrame.Display.Terminal.Colours, + DataFrame.Internal.DataFrame, + DataFrame.Internal.Row, + DataFrame.Errors, + DataFrame.Operations.Core, + DataFrame.Operations.Subset, + DataFrame.Operations.Sorting, + DataFrame.Operations.Statistics, + DataFrame.Operations.Transformations, + DataFrame.Operations.Typing, + DataFrame.Operations.Aggregation, + DataFrame.Display.Terminal.Plot, + DataFrame.IO.CSV, + DataFrame.Operations.Join, + DataFrame.Operations.Merge, + DataFrame.IO.Parquet, + DataFrame.Lazy.IO.CSV, + DataFrame.Functions + build-depends: base >= 4.17.2.0 && < 4.22, + array ^>= 0.5, + attoparsec >= 0.12 && <= 0.14.4, + bytestring >= 0.11 && <= 0.12.2.0, + containers >= 0.6.7 && < 0.8, + directory >= 1.3.0.0 && <= 1.3.9.0, + hashable >= 1.2 && <= 1.5.0.0, + snappy >= 0.2.0.0 && <= 0.2.0.4, + statistics >= 0.16.2.1 && <= 0.16.3.0, + template-haskell >= 2.0 && <= 2.30, + text >= 2.0 && <= 2.1.2, + time >= 1.12 && <= 1.14, + vector ^>= 0.13, + vector-algorithms ^>= 0.9, + zstd >= 0.1.2.0 && <= 0.1.3.0 + hs-source-dirs: examples, + src + default-language: Haskell2010 + executable dataframe main-is: Main.hs other-modules: DataFrame, @@ -85,8 +224,11 @@ DataFrame.Operations.Aggregation, DataFrame.Display.Terminal.Plot, DataFrame.IO.CSV, + DataFrame.Operations.Join, + DataFrame.Operations.Merge, DataFrame.IO.Parquet, - DataFrame.Lazy.IO.CSV + DataFrame.Lazy.IO.CSV, + DataFrame.Functions build-depends: base >= 4.17.2.0 && < 4.22, array ^>= 0.5, attoparsec >= 0.12 && <= 0.14.4, @@ -94,8 +236,10 @@ containers >= 0.6.7 && < 0.8, directory >= 1.3.0.0 && <= 1.3.9.0, hashable >= 1.2 && <= 1.5.0.0, + random >= 1 && <= 1.3.1, snappy >= 0.2.0.0 && <= 0.2.0.4, statistics >= 0.16.2.1 && <= 0.16.3.0, + template-haskell >= 2.0 && <= 2.30, text >= 2.0 && <= 2.1.2, time >= 1.12 && <= 1.14, vector ^>= 0.13, @@ -105,13 +249,16 @@ src default-language: Haskell2010 + benchmark dataframe-benchmark type: exitcode-stdio-1.0 main-is: Main.hs hs-source-dirs: benchmark build-depends: base >= 4.17.2.0 && < 4.22, criterion >= 1 && <= 1.6.4.0, + process >= 1.6, text >= 2.0 && <= 2.1.2, + time >= 1.12, random >= 1 && <= 1.3.1, vector ^>= 0.13, dataframe diff --git a/examples/CaliforniaHousing.hs b/examples/CaliforniaHousing.hs new file mode 100644 --- /dev/null +++ b/examples/CaliforniaHousing.hs @@ -0,0 +1,14 @@ +{-# LANGUAGE OverloadedStrings #-} +module Main where + +import qualified DataFrame as D + +main :: IO () +main = do + parsed <- D.readCsv "./data/housing.csv" + + print $ D.columnInfo parsed + + print $ D.take 5 parsed + + D.plotHistograms D.PlotAll D.VerticalHistogram parsed diff --git a/examples/Chipotle.hs b/examples/Chipotle.hs new file mode 100644 --- /dev/null +++ b/examples/Chipotle.hs @@ -0,0 +1,75 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} +module Main where + +import qualified DataFrame as D +import qualified DataFrame.Functions as F +import qualified Data.Text as T + +import DataFrame ((|>)) + +main :: IO () +main = do + raw <- D.readTsv "./data/chipotle.tsv" + print $ D.dimensions raw + + -- -- Sampling the dataframe + print $ D.take 5 raw + + -- Transform the data from a raw string into + -- respective types (throws error on failure) + let df = + raw + -- Change a specfic order ID + |> D.applyWhere (== (1 ::Int)) "order_id" (+ (2 :: Int)) "quantity" + -- Index based change. + |> D.applyAtIndex 0 ((\n -> n - 2) :: Int -> Int) "quantity" + -- Custom parsing: drop dollar sign and parse price as double + |> D.apply (D.readValue @Double . T.drop 1) "item_price" + + -- sample the dataframe. + print $ D.take 10 df + + -- Create a total_price column that is quantity * item_price + let withTotalPrice = D.derive "total_price" (F.lift fromIntegral (F.col @Int "quantity") * F.col @Double"item_price") df + + -- sample a filtered subset of the dataframe + putStrLn "Sample dataframe" + print $ + withTotalPrice + |> D.select ["quantity", "item_name", "item_price", "total_price"] + |> D.filter "total_price" ((100.0 :: Double) <) + |> D.take 10 + + -- Check how many chicken burritos were ordered. + -- There are two ways to checking how many chicken burritos + -- were ordered. + let searchTerm = "Chicken Burrito" :: T.Text + + print $ + df + |> D.select ["item_name", "quantity"] + -- It's more efficient to filter before grouping. + |> D.filter "item_name" (searchTerm ==) + |> D.groupBy ["item_name"] + |> D.aggregate [ (F.sum (F.col @Int "quantity")) `F.as` "sum" + , (F.maximum (F.col @Int "quantity")) `F.as` "max" + , (F.mean (F.col @Int "quantity")) `F.as` "mean"] + |> D.sortBy D.Descending ["sum"] + + -- Similarly, we can aggregate quantities by all rows. + print $ + df + |> D.select ["item_name", "quantity"] + |> D.groupBy ["item_name"] + |> D.aggregate [ (F.sum (F.col @Int "quantity")) `F.as` "sum" + , (F.maximum (F.col @Int "quantity")) `F.as` "maximum" + , (F.mean (F.col @Int "quantity")) `F.as` "mean"] + |> D.take 10 + + let firstOrder = + withTotalPrice + |> D.filterBy (maybe False (T.isInfixOf "Guacamole")) "choice_description" + |> D.filterBy (("Chicken Bowl" :: T.Text) ==) "item_name" + + print $ D.take 10 firstOrder diff --git a/examples/OneBillionRowChallenge.hs b/examples/OneBillionRowChallenge.hs new file mode 100644 --- /dev/null +++ b/examples/OneBillionRowChallenge.hs @@ -0,0 +1,20 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} +module Main where + +import qualified DataFrame as D +import qualified DataFrame.Functions as F + +import DataFrame ((|>)) + +main :: IO () +main = do + parsed <- D.readSeparated ';' D.defaultOptions "./data/measurements.txt" + let measurement = (F.col @Double "Measurement") + print $ + parsed + |> D.groupBy ["City"] + |> D.aggregate [ (F.minimum measurement) `F.as` "minimum" + , (F.mean measurement) `F.as` "mean" + , (F.maximum measurement) `F.as` "maximum"] + |> D.sortBy D.Ascending ["City"] diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -1,6 +1,8 @@ +{-# LANGUAGE OverloadedStrings #-} module DataFrame ( module D, - (|>) + (|>), + printSessionSchema ) where @@ -12,6 +14,8 @@ import DataFrame.Internal.Row as D hiding (mkRowRep) import DataFrame.Errors as D import DataFrame.Operations.Core as D +import DataFrame.Operations.Merge as D +import DataFrame.Operations.Join as D import DataFrame.Operations.Subset as D import DataFrame.Operations.Sorting as D import DataFrame.Operations.Statistics as D @@ -24,5 +28,21 @@ import DataFrame.IO.Parquet as D import Data.Function +import Data.List +import qualified Data.Text as T +import qualified Data.Text.IO as T (|>) = (&) + +printSessionSchema :: D.DataFrame -> IO () +printSessionSchema df = T.putStrLn $ let + (lhs, rhs) = foldr go ([], []) (D.columnNames df) + columnRep name = let + colType = T.pack (D.columnTypeString (D.unsafeGetColumn name df)) + in "F.col @(" <> colType <> ") \"" <> name <> "\"" + go name (l, r) = (T.toLower name:l, columnRep name:r) + in T.unlines [":{", "{-# LANGUAGE TypeApplications #-}", + "import qualified DataFrame.Functions as F", + "import Data.Text (Text)", + "(" <> T.intercalate "," lhs <> ")" <> " = " <> "(" <> T.intercalate "," rhs <> ")", + ":}"] diff --git a/src/DataFrame/Display/Terminal/Plot.hs b/src/DataFrame/Display/Terminal/Plot.hs --- a/src/DataFrame/Display/Terminal/Plot.hs +++ b/src/DataFrame/Display/Terminal/Plot.hs @@ -55,9 +55,8 @@ plotForColumnBy col cname byColumn plotColumn orientation df -- Plot code adapted from: https://alexwlchan.net/2018/ascii-bar-charts/ -plotForColumnBy :: HasCallStack => T.Text -> T.Text -> Maybe Column -> Maybe Column -> HistogramOrientation -> DataFrame -> IO () -plotForColumnBy _ _ Nothing _ _ _ = return () -plotForColumnBy byCol cname (Just (BoxedColumn (byColumn :: V.Vector a))) (Just (BoxedColumn (plotColumn :: V.Vector b))) orientation df = do +plotForColumnBy :: HasCallStack => T.Text -> T.Text -> Column -> Column -> HistogramOrientation -> DataFrame -> IO () +plotForColumnBy byCol cname (BoxedColumn (byColumn :: V.Vector a)) (BoxedColumn (plotColumn :: V.Vector b)) orientation df = do let zipped = VG.zipWith (\left right -> (show left, show right)) plotColumn byColumn let counts = countOccurrences zipped if null counts || length counts > 20 @@ -65,7 +64,7 @@ else case orientation of VerticalHistogram -> error "Vertical histograms aren't yet supported" HorizontalHistogram -> plotGivenCounts' cname counts -plotForColumnBy byCol cname (Just (UnboxedColumn byColumn)) (Just (BoxedColumn plotColumn)) orientation df = do +plotForColumnBy byCol cname (UnboxedColumn byColumn) (BoxedColumn plotColumn) orientation df = do let zipped = VG.zipWith (\left right -> (show left, show right)) plotColumn (V.convert byColumn) let counts = countOccurrences zipped if null counts || length counts > 20 @@ -73,7 +72,7 @@ else case orientation of VerticalHistogram -> error "Vertical histograms aren't yet supported" HorizontalHistogram -> plotGivenCounts' cname counts -plotForColumnBy byCol cname (Just (BoxedColumn byColumn)) (Just (UnboxedColumn plotColumn)) orientation df = do +plotForColumnBy byCol cname (BoxedColumn byColumn) (UnboxedColumn plotColumn) orientation df = do let zipped = VG.zipWith (\left right -> (show left, show right)) (V.convert plotColumn) (V.convert byColumn) let counts = countOccurrences zipped if null counts || length counts > 20 @@ -81,7 +80,7 @@ else case orientation of -- VerticalHistogram -> plotVerticalGivenCounts cname counts HorizontalHistogram -> plotGivenCounts' cname counts -plotForColumnBy byCol cname (Just (UnboxedColumn byColumn)) (Just (UnboxedColumn plotColumn)) orientation df = do +plotForColumnBy byCol cname (UnboxedColumn byColumn) (UnboxedColumn plotColumn) orientation df = do let zipped = VG.zipWith (\left right -> (show left, show right)) (V.convert plotColumn) (V.convert byColumn) let counts = countOccurrences zipped if null counts || length counts > 20 @@ -93,9 +92,8 @@ plotForColumnBy _ _ _ _ _ _ = return () -- Plot code adapted from: https://alexwlchan.net/2018/ascii-bar-charts/ -plotForColumn :: HasCallStack => T.Text -> Maybe Column -> HistogramOrientation -> DataFrame -> IO () -plotForColumn _ Nothing _ _ = return () -plotForColumn cname (Just (BoxedColumn (column :: V.Vector a))) orientation df = do +plotForColumn :: HasCallStack => T.Text -> Column -> HistogramOrientation -> DataFrame -> IO () +plotForColumn cname (BoxedColumn (column :: V.Vector a)) orientation df = do let repa :: Ref.TypeRep a = Ref.typeRep @a repText :: Ref.TypeRep T.Text = Ref.typeRep @T.Text repString :: Ref.TypeRep String = Ref.typeRep @String @@ -110,7 +108,7 @@ else case orientation of VerticalHistogram -> plotVerticalGivenCounts cname counts HorizontalHistogram -> plotGivenCounts cname counts -plotForColumn cname (Just (UnboxedColumn (column :: VU.Vector a))) orientation df = do +plotForColumn cname (UnboxedColumn (column :: VU.Vector a)) orientation df = do let repa :: Ref.TypeRep a = Ref.typeRep @a repText :: Ref.TypeRep T.Text = Ref.typeRep @T.Text repString :: Ref.TypeRep String = Ref.typeRep @String diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Functions.hs @@ -0,0 +1,110 @@ +{-# LANGUAGE ExplicitNamespaces #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE InstanceSigs #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE TemplateHaskell #-} +module DataFrame.Functions where + +import DataFrame.Internal.Column +import DataFrame.Internal.DataFrame (DataFrame(..), unsafeGetColumn) +import DataFrame.Internal.Expression (Expr(..), UExpr(..)) + +import Control.Monad +import Data.Function +import qualified Data.List as L +import qualified Data.Map as M +import qualified Data.Text as T +import qualified Data.Vector.Generic as VG +import qualified Data.Vector.Unboxed as VU +import qualified Data.Vector as VB +import Language.Haskell.TH +import qualified Language.Haskell.TH.Syntax as TH + +col :: Columnable a => T.Text -> Expr a +col = Col + +as :: Columnable a => Expr a -> T.Text -> (T.Text, UExpr) +as expr name = (name, Wrap expr) + +lit :: Columnable a => a -> Expr a +lit = Lit + +lift :: (Columnable a, Columnable b) => (a -> b) -> Expr a -> Expr b +lift = Apply "udf" + +lift2 :: (Columnable c, Columnable b, Columnable a) => (c -> b -> a) -> Expr c -> Expr b -> Expr a +lift2 = BinOp "udf" + +eq :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool +eq = BinOp "eq" (==) + +lt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool +lt = BinOp "lt" (<) + +gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool +gt = BinOp "gt" (>) + +leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool +leq = BinOp "leq" (<=) + +geq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool +geq = BinOp "geq" (>=) + +count :: Columnable a => Expr a -> Expr Int +count (Col name) = GeneralAggregate name "count" VG.length +count _ = error "Argument can only be a column reference not an unevaluated expression" + +minimum :: Columnable a => Expr a -> Expr a +minimum (Col name) = ReductionAggregate name "minimum" VG.minimum + +maximum :: Columnable a => Expr a -> Expr a +maximum (Col name) = ReductionAggregate name "maximum" VG.maximum + +sum :: (Columnable a, Num a, VU.Unbox a) => Expr a -> Expr a +sum (Col name) = NumericAggregate name "sum" VG.sum + +mean :: (Columnable a, Num a, VU.Unbox a) => Expr a -> Expr Double +mean (Col name) = let + mean' samp = let + (!total, !n) = VG.foldl' (\(!total, !n) v -> (total + v, n + 1)) (0 :: Double, 0 :: Int) samp + in total / fromIntegral n + in NumericAggregate name "mean" mean' + +typeFromString :: String -> Q Type +typeFromString s = do + maybeType <- lookupTypeName s + case maybeType of + Just name -> return (ConT name) + Nothing -> do + if "Maybe " `L.isPrefixOf` s + then do + let innerType = drop 6 s + inner <- typeFromString innerType + return (AppT (ConT ''Maybe) inner) + else if "Either " `L.isPrefixOf` s + then do + let (left: right:_) = tail (words s) + lhs <- typeFromString left + rhs <- typeFromString right + return (AppT (AppT (ConT ''Either) lhs) rhs) + else fail $ "Unsupported type: " ++ s + +declareColumns :: DataFrame -> DecsQ +declareColumns df = let + names = (map fst . L.sortBy (compare `on` snd). M.toList . columnIndices) df + types = map (columnTypeString . (`unsafeGetColumn` df)) names + specs = zip names types + in fmap concat $ forM specs $ \(nm, tyStr) -> do + ty <- typeFromString tyStr + let n = mkName (T.unpack nm) + sig <- sigD n [t| Expr $(pure ty) |] + val <- valD (varP n) (normalB [| col $(TH.lift nm) |]) [] + pure [sig, val] diff --git a/src/DataFrame/IO/CSV.hs b/src/DataFrame/IO/CSV.hs --- a/src/DataFrame/IO/CSV.hs +++ b/src/DataFrame/IO/CSV.hs @@ -1,4 +1,3 @@ -{-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} @@ -6,7 +5,6 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE RankNTypes #-} -{-# LANGUAGE Strict #-} module DataFrame.IO.CSV where import qualified Data.ByteString.Char8 as C @@ -104,9 +102,8 @@ cols <- V.mapM (freezeColumn mutableCols nulls' opts) (V.generate numColumns id) return $ DataFrame { columns = cols, - freeIndices = [], columnIndices = M.fromList (zip columnNames [0..]), - dataframeDimensions = (maybe 0 columnLength (cols V.! 0), V.length cols) + dataframeDimensions = (maybe 0 columnLength (cols V.!? 0), V.length cols) } {-# INLINE readSeparated #-} @@ -160,10 +157,10 @@ {-# INLINE writeValue #-} -- | Freezes a mutable vector into an immutable one, trimming it to the actual row count. -freezeColumn :: VM.IOVector Column -> V.Vector [(Int, T.Text)] -> ReadOptions -> Int -> IO (Maybe Column) +freezeColumn :: VM.IOVector Column -> V.Vector [(Int, T.Text)] -> ReadOptions -> Int -> IO Column freezeColumn mutableCols nulls opts colIndex = do col <- VM.unsafeRead mutableCols colIndex - Just <$> freezeColumn' (nulls V.! colIndex) col + freezeColumn' (nulls V.! colIndex) col {-# INLINE freezeColumn #-} parseSep :: Char -> T.Text -> [T.Text] @@ -215,7 +212,7 @@ countRows :: Char -> FilePath -> IO Int countRows c path = withFile path ReadMode $! go 0 "" where - go !n !input h = do + go n input h = do isEOF <- hIsEOF h if isEOF && input == mempty then pure n @@ -250,8 +247,7 @@ getRowAsText df i = V.ifoldr go [] (columns df) where indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df)) - go k Nothing acc = acc - go k (Just (BoxedColumn (c :: V.Vector a))) acc = case c V.!? i of + go k (BoxedColumn (c :: V.Vector a)) acc = case c V.!? i of Just e -> textRep : acc where textRep = case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> e @@ -272,7 +268,7 @@ ++ " has less items than " ++ "the other columns at index " ++ show i - go k (Just (UnboxedColumn c)) acc = case c VU.!? i of + go k (UnboxedColumn c) acc = case c VU.!? i of Just e -> T.pack (show e) : acc Nothing -> error $ @@ -281,7 +277,7 @@ ++ " has less items than " ++ "the other columns at index " ++ show i - go k (Just (OptionalColumn (c :: V.Vector (Maybe a)))) acc = case c V.!? i of + go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of Just e -> textRep : acc where textRep = case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> fromMaybe "Nothing" e diff --git a/src/DataFrame/Internal/Column.hs b/src/DataFrame/Internal/Column.hs --- a/src/DataFrame/Internal/Column.hs +++ b/src/DataFrame/Internal/Column.hs @@ -4,7 +4,6 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE Strict #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} @@ -233,7 +232,7 @@ @ > import qualified Data.Vector.Unboxed as V -> fromVector (V.fromList [(1 :: Int), 2, 3, 4]) +> fromUnboxedVector (V.fromList [(1 :: Int), 2, 3, 4]) [1,2,3,4] @ -} @@ -254,17 +253,19 @@ => [a] -> Column fromList = toColumnRep @(KindOf a) . VB.fromList - +-- | Type-level boolean for constraint/type comparison. data SBool (b :: Bool) where STrue :: SBool 'True SFalse :: SBool 'False +-- | The runtime witness for our type-level branching. class SBoolI (b :: Bool) where - sbool :: SBool b -- the run-time witness + sbool :: SBool b instance SBoolI 'True where sbool = STrue instance SBoolI 'False where sbool = SFalse +-- | Type-level function to determine whether or not a type is unboxa sUnbox :: forall a. SBoolI (Unboxable a) => SBool (Unboxable a) sUnbox = sbool @(Unboxable a) @@ -377,6 +378,26 @@ getIndicesUnboxed indices xs = VU.generate (VU.length indices) (\i -> xs VU.! (indices VU.! i)) {-# INLINE getIndicesUnboxed #-} +findIndices :: forall a. (Columnable a) + => (a -> Bool) + -> Column + -> Maybe (VU.Vector Int) +findIndices pred (BoxedColumn (column :: VB.Vector b)) = do + Refl <- testEquality (typeRep @a) (typeRep @b) + pure $ VG.convert (VG.findIndices pred column) +findIndices pred (UnboxedColumn (column :: VU.Vector b)) = do + Refl <- testEquality (typeRep @a) (typeRep @b) + pure $ VG.findIndices pred column +findIndices pred (OptionalColumn (column :: VB.Vector (Maybe b))) = do + Refl <- testEquality (typeRep @a) (typeRep @(Maybe b)) + pure $ VG.convert (VG.findIndices pred column) +findIndices pred (GroupedBoxedColumn (column :: VB.Vector b)) = do + Refl <- testEquality (typeRep @a) (typeRep @b) + pure $ VG.convert (VG.findIndices pred column) +findIndices pred (GroupedUnboxedColumn (column :: VB.Vector b)) = do + Refl <- testEquality (typeRep @a) (typeRep @b) + pure $ VG.convert (VG.findIndices pred column) + -- | An internal function that returns a vector of how indexes change after a column is sorted. sortedIndexes :: Bool -> Column -> VU.Vector Int sortedIndexes asc (BoxedColumn column ) = runST $ do @@ -564,27 +585,37 @@ -- | Fills the end of a column, up to n, with Nothing. Does nothing if column has length greater than n. expandColumn :: Int -> Column -> Column -expandColumn n (OptionalColumn col) = OptionalColumn $ col <> VB.replicate n Nothing +expandColumn n (OptionalColumn col) = OptionalColumn $ col <> VB.replicate (n - VG.length col) Nothing expandColumn n column@(BoxedColumn col) - | n < VG.length col = OptionalColumn $ VB.map Just col <> VB.replicate n Nothing + | n > VG.length col = OptionalColumn $ VB.map Just col <> VB.replicate (n - VG.length col) Nothing | otherwise = column expandColumn n column@(UnboxedColumn col) - | n < VG.length col = OptionalColumn $ VB.map Just (VU.convert col) <> VB.replicate n Nothing + | n > VG.length col = OptionalColumn $ VB.map Just (VU.convert col) <> VB.replicate (n - VG.length col) Nothing | otherwise = column expandColumn n column@(GroupedBoxedColumn col) - | n < VG.length col = GroupedBoxedColumn $ col <> VB.replicate n VB.empty + | n > VG.length col = GroupedBoxedColumn $ col <> VB.replicate (n - VG.length col) VB.empty | otherwise = column expandColumn n column@(GroupedUnboxedColumn col) - | n < VG.length col = GroupedUnboxedColumn $ col <> VB.replicate n VU.empty + | n > VG.length col = GroupedUnboxedColumn $ col <> VB.replicate (n - VG.length col) VU.empty | otherwise = column -- | Fills the beginning of a column, up to n, with Nothing. Does nothing if column has length greater than n. leftExpandColumn :: Int -> Column -> Column -leftExpandColumn n (OptionalColumn col) = OptionalColumn $ VB.replicate n Nothing <> col -leftExpandColumn n (BoxedColumn col) = OptionalColumn $ VB.replicate n Nothing <> VB.map Just col -leftExpandColumn n (UnboxedColumn col) = OptionalColumn $ VB.replicate n Nothing <> VB.map Just (VU.convert col) -leftExpandColumn n (GroupedBoxedColumn col) = GroupedBoxedColumn $ VB.replicate n VB.empty <> col -leftExpandColumn n (GroupedUnboxedColumn col) = GroupedUnboxedColumn $ VB.replicate n VU.empty <> col +leftExpandColumn n column@(OptionalColumn col) + | n > VG.length col = OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> col + | otherwise = column +leftExpandColumn n column@(BoxedColumn col) + | n > VG.length col = OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> VG.map Just col + | otherwise = column +leftExpandColumn n column@(UnboxedColumn col) + | n > VG.length col = OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> VG.map Just (VU.convert col) + | otherwise = column +leftExpandColumn n column@(GroupedBoxedColumn col) + | n > VG.length col = GroupedBoxedColumn $ VG.replicate (n - VG.length col) VB.empty <> col + | otherwise = column +leftExpandColumn n column@(GroupedUnboxedColumn col) + | n > VG.length col = GroupedUnboxedColumn $ VG.replicate (n - VG.length col) VU.empty <> col + | otherwise = column -- | Concatenates two columns. concatColumns :: Column -> Column -> Maybe Column @@ -617,42 +648,44 @@ exception: ... -} toVector :: forall a . Columnable a => Column -> VB.Vector a -toVector = toVectorWithLabel "toVector" +toVector xs = case toVectorSafe xs of + Left err -> throw err + Right val -> val --- | An internal version of toVector that takes the calling function as an extra argument. -toVectorWithLabel :: forall a . Columnable a => String -> Column -> VB.Vector a -toVectorWithLabel label column@(OptionalColumn (col :: VB.Vector b)) = +-- | A safe version of toVector that returns an Either type. +toVectorSafe :: forall a . Columnable a => Column -> Either DataFrameException (VB.Vector a) +toVectorSafe column@(OptionalColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> col - Nothing -> throw $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just label - , errorColumnName = Nothing}) -toVectorWithLabel label (BoxedColumn (col :: VB.Vector b)) = + Just Refl -> Right col + Nothing -> Left $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "toVectorSafe" + , errorColumnName = Nothing}) +toVectorSafe (BoxedColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> col - Nothing -> throw $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just label - , errorColumnName = Nothing}) -toVectorWithLabel label (UnboxedColumn (col :: VU.Vector b)) = + Just Refl -> Right col + Nothing -> Left $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "toVectorSafe" + , errorColumnName = Nothing}) +toVectorSafe (UnboxedColumn (col :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> VB.convert col - Nothing -> throw $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just label - , errorColumnName = Nothing}) -toVectorWithLabel label (GroupedBoxedColumn (col :: VB.Vector b)) = + Just Refl -> Right $ VB.convert col + Nothing -> Left $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "toVectorSafe" + , errorColumnName = Nothing}) +toVectorSafe (GroupedBoxedColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> col - Nothing -> throw $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just label - , errorColumnName = Nothing}) -toVectorWithLabel label (GroupedUnboxedColumn (col :: VB.Vector b)) = + Just Refl -> Right col + Nothing -> Left $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "toVectorSafe" + , errorColumnName = Nothing}) +toVectorSafe (GroupedUnboxedColumn (col :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> col - Nothing -> throw $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just label - , errorColumnName = Nothing}) + Just Refl -> Right col + Nothing -> Left $ TypeMismatchException (MkTypeErrorContext { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "toVectorSafe" + , errorColumnName = Nothing}) diff --git a/src/DataFrame/Internal/DataFrame.hs b/src/DataFrame/Internal/DataFrame.hs --- a/src/DataFrame/Internal/DataFrame.hs +++ b/src/DataFrame/Internal/DataFrame.hs @@ -4,7 +4,6 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE GADTs #-} -{-# LANGUAGE Strict #-} {-# LANGUAGE FlexibleContexts #-} module DataFrame.Internal.DataFrame where @@ -25,11 +24,9 @@ data DataFrame = DataFrame { -- | Our main data structure stores a dataframe as -- a vector of columns. This improv - columns :: V.Vector (Maybe Column), + columns :: V.Vector Column, -- | Keeps the column names in the order they were inserted in. columnIndices :: M.Map T.Text Int, - -- | Next free index that we insert a column into. - freeIndices :: [Int], dataframeDimensions :: (Int, Int) } @@ -46,13 +43,12 @@ asText d properMarkdown = let header = "index" : map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d)) types = V.toList $ V.filter (/= "") $ V.map getType (columns d) - getType :: Maybe Column -> T.Text - getType Nothing = "" - getType (Just (BoxedColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a) - getType (Just (UnboxedColumn (column :: VU.Vector a))) = T.pack $ show (typeRep @a) - getType (Just (OptionalColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a) - getType (Just (GroupedBoxedColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a) - getType (Just (GroupedUnboxedColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a) + getType :: Column -> T.Text + getType (BoxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a) + getType (UnboxedColumn (column :: VU.Vector a)) = T.pack $ show (typeRep @a) + getType (OptionalColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a) + getType (GroupedBoxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a) + getType (GroupedUnboxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a) -- Separate out cases dynamically so we don't end up making round trip string -- copies. get :: Maybe Column -> V.Vector T.Text @@ -67,7 +63,7 @@ get (Just (GroupedUnboxedColumn column)) = V.map (T.pack . show) column getTextColumnFromFrame df (i, name) = if i == 0 then V.fromList (map (T.pack . show) [0..(fst (dataframeDimensions df) - 1)]) - else get $ (V.!) (columns d) ((M.!) (columnIndices d) name) + else get $ (V.!?) (columns d) ((M.!) (columnIndices d) name) rows = transpose $ zipWith (curry (V.toList . getTextColumnFromFrame d)) [0..] header @@ -75,24 +71,17 @@ -- | O(1) Creates an empty dataframe empty :: DataFrame -empty = DataFrame {columns = V.replicate initialColumnSize Nothing, +empty = DataFrame {columns = V.empty, columnIndices = M.empty, - freeIndices = [0..(initialColumnSize - 1)], dataframeDimensions = (0, 0) } -initialColumnSize :: Int -initialColumnSize = 8 - getColumn :: T.Text -> DataFrame -> Maybe Column getColumn name df = do i <- columnIndices df M.!? name - join $ columns df V.!? i + columns df V.!? i -null :: DataFrame -> Bool -null df = dataframeDimensions df == (0, 0) +unsafeGetColumn :: T.Text -> DataFrame -> Column +unsafeGetColumn name df = columns df V.! (columnIndices df M.! name) -metadata :: DataFrame -> String -metadata df = show (columnIndices df) ++ "\n" ++ - show (V.map (fmap columnVersionString) (columns df)) ++ "\n" ++ - show (freeIndices df) ++ "\n" ++ - show (dataframeDimensions df) +null :: DataFrame -> Bool +null df = V.null (columns df) diff --git a/src/DataFrame/Internal/Expression.hs b/src/DataFrame/Internal/Expression.hs --- a/src/DataFrame/Internal/Expression.hs +++ b/src/DataFrame/Internal/Expression.hs @@ -4,7 +4,6 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE StrictData #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} @@ -21,6 +20,7 @@ import qualified Data.Text as T import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU +import qualified Data.Vector.Generic as VG import Type.Reflection (typeRep) import DataFrame.Errors (DataFrameException(ColumnNotFoundException)) import Control.Exception (throw) @@ -29,9 +29,42 @@ data Expr a where Col :: Columnable a => T.Text -> Expr a Lit :: Columnable a => a -> Expr a - Apply :: (Columnable a, Columnable b) => T.Text -> (b -> a) -> Expr b -> Expr a - BinOp :: (Columnable c, Columnable b, Columnable a) => T.Text -> (c -> b -> a) -> Expr c -> Expr b -> Expr a + Apply :: (Columnable a, + Columnable b) + => T.Text -- Operation name + -> (b -> a) + -> Expr b + -> Expr a + BinOp :: (Columnable c, + Columnable b, + Columnable a) + => T.Text -- operation name + -> (c -> b -> a) + -> Expr c + -> Expr b + -> Expr a + GeneralAggregate :: (Columnable a) + => T.Text -- Column name + -> T.Text -- Operation name + -> (forall v b. (VG.Vector v b, Columnable b) => v b -> a) + -> Expr a + ReductionAggregate :: (Columnable a) + => T.Text -- Column name + -> T.Text -- Operation name + -> (forall v a. (VG.Vector v a, Columnable a) => v a -> a) + -> Expr a + NumericAggregate :: (Columnable a, + Columnable b, + Num a, + Num b) + => T.Text -- Column name + -> T.Text -- Operation name + -> (VU.Vector b -> a) + -> Expr a +data UExpr where + Wrap :: Columnable a => Expr a -> UExpr + interpret :: forall a . (Columnable a) => DataFrame -> Expr a -> TypedColumn a interpret df (Lit value) = TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value interpret df (Col name) = case getColumn name df of @@ -45,6 +78,32 @@ (TColumn left') = interpret @c df left (TColumn right') = interpret @d df right in TColumn $ fromMaybe (error "mapColumn returned nothing") (zipWithColumns f left' right') +interpret df (GeneralAggregate name op (f :: forall v b. (VG.Vector v b, Columnable b) => v b -> c)) = case getColumn name df of + Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) + Just (GroupedBoxedColumn col) -> TColumn $ fromVector $ VG.map f col + Just (GroupedUnboxedColumn col) -> TColumn $ fromVector $ VG.map f col + Just (GroupedOptionalColumn col) -> TColumn $ fromVector $ VG.map f col + _ -> error "" +interpret df (ReductionAggregate name op (f :: forall v a. (VG.Vector v a, Columnable a) => v a -> a)) = case getColumn name df of + Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) + Just (GroupedBoxedColumn col) -> TColumn $ fromVector $ VG.map f col + Just (GroupedUnboxedColumn col) -> TColumn $ fromVector $ VG.map f col + Just (GroupedOptionalColumn col) -> TColumn $ fromVector $ VG.map f col + _ -> error "" +interpret df (NumericAggregate name op (f :: VU.Vector b -> c)) = case getColumn name df of + Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) + Just (GroupedUnboxedColumn (col :: V.Vector (VU.Vector d))) -> case testEquality (typeRep @b) (typeRep @d) of + Just Refl -> TColumn $ fromVector $ VG.map f col + -- Do the matching trick here. + Nothing -> case testEquality (typeRep @d) (typeRep @Int) of + Just Refl -> case testEquality (typeRep @b) (typeRep @Double) of + Just Refl -> TColumn $ fromVector $ VG.map (f . (VG.map fromIntegral)) col + Nothing -> error $ "Column not a number: " ++ (T.unpack name) + Nothing -> case testEquality (typeRep @d) (typeRep @Double) of + Just Refl -> case testEquality (typeRep @b) (typeRep @Int) of + Just Refl -> TColumn $ fromVector $ VG.map (f . (VG.map round)) col + Nothing -> error $ "Column not a number: " ++ (T.unpack name) + _ -> error "Cannot apply numeric aggregation to boxed column" instance (Num a, Columnable a) => Num (Expr a) where (+) :: Expr a -> Expr a -> Expr a @@ -107,30 +166,3 @@ show (Lit value) = show value show (Apply name f value) = T.unpack name ++ "(" ++ show value ++ ")" show (BinOp name f a b) = T.unpack name ++ "(" ++ show a ++ ", " ++ show b ++ ")" - -col :: Columnable a => T.Text -> Expr a -col = Col - -lit :: Columnable a => a -> Expr a -lit = Lit - -lift :: (Columnable a, Columnable b) => (a -> b) -> Expr a -> Expr b -lift = Apply "udf" - -lift2 :: (Columnable c, Columnable b, Columnable a) => (c -> b -> a) -> Expr c -> Expr b -> Expr a -lift2 = BinOp "udf" - -eq :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool -eq = BinOp "eq" (==) - -lt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool -lt = BinOp "lt" (<) - -gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool -gt = BinOp "gt" (>) - -leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool -leq = BinOp "leq" (<=) - -geq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool -geq = BinOp "geq" (>=) diff --git a/src/DataFrame/Internal/Parsing.hs b/src/DataFrame/Internal/Parsing.hs --- a/src/DataFrame/Internal/Parsing.hs +++ b/src/DataFrame/Internal/Parsing.hs @@ -1,5 +1,4 @@ {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE Strict #-} module DataFrame.Internal.Parsing where import qualified Data.ByteString.Char8 as C diff --git a/src/DataFrame/Internal/Types.hs b/src/DataFrame/Internal/Types.hs --- a/src/DataFrame/Internal/Types.hs +++ b/src/DataFrame/Internal/Types.hs @@ -8,7 +8,6 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeOperators #-} -{-# LANGUAGE Strict #-} module DataFrame.Internal.Types where import Data.Int ( Int8, Int16, Int32, Int64 ) diff --git a/src/DataFrame/Lazy/IO/CSV.hs b/src/DataFrame/Lazy/IO/CSV.hs --- a/src/DataFrame/Lazy/IO/CSV.hs +++ b/src/DataFrame/Lazy/IO/CSV.hs @@ -1,4 +1,3 @@ -{-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} @@ -6,7 +5,6 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE RankNTypes #-} -{-# LANGUAGE Strict #-} module DataFrame.Lazy.IO.CSV where import qualified Data.ByteString.Char8 as C @@ -100,8 +98,7 @@ -- TODO: this isn't robust but in so far as this is a guess anyway -- it's probably fine. But we should probably sample n rows and pick -- the most likely type from the sample. - -- dataRow <- map T.strip . parseSep c . (<>) (leftOver opts) <$> TIO.hGetLine handle - (!dataRow, !remainder) <- readSingleLine c (leftOver opts) handle + (dataRow, remainder) <- readSingleLine c (leftOver opts) handle -- This array will track the indices of all null values for each column. -- If any exist then the column will be an optional type. @@ -111,7 +108,7 @@ getInitialDataVectors numRows mutableCols dataRow -- Read rows into the mutable vectors - (!unconsumed, !r) <- fillColumns numRows c mutableCols nullIndices remainder handle + (unconsumed, r) <- fillColumns numRows c mutableCols nullIndices remainder handle -- Freeze the mutable vectors into immutable ones nulls' <- V.unsafeFreeze nullIndices @@ -120,9 +117,8 @@ return (DataFrame { columns = cols, - freeIndices = [], columnIndices = M.fromList (zip columnNames [0..]), - dataframeDimensions = (maybe 0 columnLength (cols V.! 0), V.length cols) + dataframeDimensions = (maybe 0 columnLength (cols V.!? 0), V.length cols) }, (pos, unconsumed, r + 1)) {-# INLINE readSeparated #-} @@ -192,10 +188,10 @@ {-# INLINE writeValue #-} -- | Freezes a mutable vector into an immutable one, trimming it to the actual row count. -freezeColumn :: VM.IOVector Column -> V.Vector [(Int, T.Text)] -> ReadOptions -> Int -> IO (Maybe Column) +freezeColumn :: VM.IOVector Column -> V.Vector [(Int, T.Text)] -> ReadOptions -> Int -> IO Column freezeColumn mutableCols nulls opts colIndex = do col <- VM.unsafeRead mutableCols colIndex - Just <$> freezeColumn' (nulls V.! colIndex) col + freezeColumn' (nulls V.! colIndex) col {-# INLINE freezeColumn #-} parseSep :: Char -> T.Text -> [T.Text] @@ -247,7 +243,7 @@ countRows :: Char -> FilePath -> IO Int countRows c path = withFile path ReadMode $! go 0 "" where - go !n !input h = do + go n input h = do isEOF <- hIsEOF h if isEOF && input == mempty then pure n @@ -282,8 +278,7 @@ getRowAsText df i = V.ifoldr go [] (columns df) where indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df)) - go k Nothing acc = acc - go k (Just (BoxedColumn (c :: V.Vector a))) acc = case c V.!? i of + go k (BoxedColumn (c :: V.Vector a)) acc = case c V.!? i of Just e -> textRep : acc where textRep = case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> e @@ -304,7 +299,7 @@ ++ " has less items than " ++ "the other columns at index " ++ show i - go k (Just (UnboxedColumn c)) acc = case c VU.!? i of + go k (UnboxedColumn c) acc = case c VU.!? i of Just e -> T.pack (show e) : acc Nothing -> error $ @@ -313,7 +308,7 @@ ++ " has less items than " ++ "the other columns at index " ++ show i - go k (Just (OptionalColumn (c :: V.Vector (Maybe a)))) acc = case c V.!? i of + go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of Just e -> textRep : acc where textRep = case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> fromMaybe "Nothing" e diff --git a/src/DataFrame/Lazy/Internal/DataFrame.hs b/src/DataFrame/Lazy/Internal/DataFrame.hs --- a/src/DataFrame/Lazy/Internal/DataFrame.hs +++ b/src/DataFrame/Lazy/Internal/DataFrame.hs @@ -3,8 +3,6 @@ {-# LANGUAGE InstanceSigs #-} {-# LANGUAGE ExistentialQuantification #-} {-# LANGUAGE AllowAmbiguousTypes #-} -{-# LANGUAGE Strict #-} -{-# LANGUAGE BangPatterns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE NumericUnderscores #-} module DataFrame.Lazy.Internal.DataFrame where @@ -56,16 +54,16 @@ let path = inputPath df totalRows <- D.countRows ',' path let batches = batchRanges totalRows (batchSize df) - (df', _) <- foldM (\(!accDf, (!pos, !unused, !r)) (!start, !end) -> do + (df', _) <- foldM (\(accDf, (pos, unused, r)) (start, end) -> do mapM_ putStr ["Scanning: ", show start, " to ", show end, " rows out of ", show totalRows, "\n"] - (!sdf, (!pos', !unconsumed, !rowsRead)) <- D.readSeparated ',' ( + (sdf, (pos', unconsumed, rowsRead)) <- D.readSeparated ',' ( D.defaultOptions { D.rowRange = Just (start, batchSize df) , D.totalRows = Just totalRows , D.seekPos = pos , D.rowsRead = r , D.leftOver = unused}) path - let !rdf = L.foldl' (flip eval) sdf (operations df) + let rdf = L.foldl' (flip eval) sdf (operations df) return (accDf <> rdf, (Just pos', unconsumed, rowsRead + r)) ) (D.empty, (Nothing, "", 0)) batches return df' diff --git a/src/DataFrame/Operations/Aggregation.hs b/src/DataFrame/Operations/Aggregation.hs --- a/src/DataFrame/Operations/Aggregation.hs +++ b/src/DataFrame/Operations/Aggregation.hs @@ -17,14 +17,19 @@ import qualified Data.Vector as V import qualified Data.Vector.Mutable as VM import qualified Data.Vector.Unboxed as VU +import qualified Data.Vector.Algorithms.Merge as VA import qualified Statistics.Quantile as SS import qualified Statistics.Sample as SS import Control.Exception (throw) import Control.Monad (foldM_) import Control.Monad.ST (runST) -import DataFrame.Internal.Column (Column(..), fromVector, getIndicesUnboxed, getIndices, Columnable) +import DataFrame.Internal.Column (Column(..), fromVector, + getIndicesUnboxed, getIndices, + Columnable, unwrapTypedColumn, + columnVersionString) import DataFrame.Internal.DataFrame (DataFrame(..), empty, getColumn) +import DataFrame.Internal.Expression import DataFrame.Internal.Parsing import DataFrame.Internal.Types import DataFrame.Errors @@ -46,29 +51,22 @@ | any (`notElem` columnNames df) names = throw $ ColumnNotFoundException (T.pack $ show $ names L.\\ columnNames df) "groupBy" (columnNames df) | otherwise = L.foldl' insertColumns initDf groupingColumns where - insertOrAdjust k v m = if MS.notMember k m then MS.insert k [v] m else MS.adjust (appendWithFrontMin v) k m - -- Create a string representation of each row. - values = V.generate (fst (dimensions df)) (mkRowRep df (S.fromList names)) - -- Create a mapping from the row representation to the list of indices that - -- have that row representation. This will allow us sortedIndexesto combine the indexes - -- where the rows are the same. - valueIndices = V.ifoldl' (\m index rowRep -> insertOrAdjust rowRep index m) M.empty values - -- Since the min is at the head this allows us to get the min in constant time and sort by it - -- That way we can recover the original order of the rows. - -- valueIndicesInitOrder = L.sortBy (compare `on` snd) $! MS.toList $ MS.map VU.head valueIndices - valueIndicesInitOrder = runST $ do - v <- VM.new (MS.size valueIndices) - foldM_ (\i idxs -> VM.write v i (VU.fromList idxs) >> return (i + 1)) 0 valueIndices - V.unsafeFreeze v + indicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` names) (columnIndices df) + rowRepresentations = VU.generate (fst (dimensions df)) (mkRowRep indicesToGroup df) + valueIndices = V.fromList $ map (VG.map fst) $ VG.groupBy (\a b -> snd a == snd b) (runST $ do + withIndexes <- VG.thaw $ VG.indexed rowRepresentations + VA.sortBy (\(a, b) (a', b') -> compare b b') withIndexes + VG.unsafeFreeze withIndexes) + -- These are the indexes of the grouping/key rows i.e the minimum elements -- of the list. - keyIndices = VU.generate (VG.length valueIndicesInitOrder) (\i -> VG.head $ valueIndicesInitOrder VG.! i) + keyIndices = VU.generate (VG.length valueIndices) (\i -> VG.minimum $ valueIndices VG.! i) -- this will be our main worker function in the fold that takes all -- indices and replaces each value in a column with a list of -- the elements with the indices where the grouped row -- values are the same. - insertColumns = groupColumns valueIndicesInitOrder df + insertColumns = groupColumns valueIndices df -- Out initial DF will just be all the grouped rows added to an -- empty dataframe. The entries are dedued and are in their -- initial order. @@ -76,174 +74,62 @@ -- All the rest of the columns that we are grouping by. groupingColumns = columnNames df L.\\ names -mkRowRep :: DataFrame -> S.Set T.Text -> Int -> Int -mkRowRep df names i = hash $ V.ifoldl' go [] (columns df) +mkRowRep :: [Int] -> DataFrame -> Int -> Int +mkRowRep groupColumnIndices df i = if length h == 1 then head h else hash h where - indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df)) - go acc k Nothing = acc - go acc k (Just (BoxedColumn (c :: V.Vector a))) = - if S.notMember (indexMap M.! k) names - then acc - else case c V.!? i of - Just e -> hash' @a e : acc - Nothing -> - error $ - "Column " - ++ T.unpack (indexMap M.! k) - ++ " has less items than " - ++ "the other columns at index " - ++ show i - go acc k (Just (OptionalColumn (c :: V.Vector (Maybe a)))) = - if S.notMember (indexMap M.! k) names - then acc - else case c V.!? i of - Just e -> hash' @(Maybe a) e : acc - Nothing -> - error $ - "Column " - ++ T.unpack (indexMap M.! k) - ++ " has less items than " - ++ "the other columns at index " - ++ show i - go acc k (Just (UnboxedColumn (c :: VU.Vector a))) = - if S.notMember (indexMap M.! k) names - then acc - else case c VU.!? i of - Just e -> hash' @a e : acc - Nothing -> - error $ - "Column " - ++ T.unpack (indexMap M.! k) - ++ " has less items than " - ++ "the other columns at index " - ++ show i + h = (map mkHash groupColumnIndices) + getHashedElem :: Column -> Int -> Int + getHashedElem (BoxedColumn (c :: V.Vector a)) j = hash' @a (c V.! j) + getHashedElem (UnboxedColumn (c :: VU.Vector a)) j = hash' @a (c VU.! j) + getHashedElem (OptionalColumn (c :: V.Vector a)) j = hash' @a (c V.! j) + getHashedElem _ _ = 0 + mkHash j = getHashedElem ((V.!) (columns df) j) i -- | This hash function returns the hash when given a non numeric type but -- the value when given a numeric. -hash' :: Columnable a => a -> Double +hash' :: Columnable a => a -> Int hash' value = case testEquality (typeOf value) (typeRep @Double) of - Just Refl -> value + Just Refl -> round $ value * 1000 Nothing -> case testEquality (typeOf value) (typeRep @Int) of - Just Refl -> fromIntegral value + Just Refl -> value Nothing -> case testEquality (typeOf value) (typeRep @T.Text) of - Just Refl -> fromIntegral $ hash value - Nothing -> fromIntegral $ hash (show value) + Just Refl -> hash value + Nothing -> hash (show value) mkGroupedColumns :: VU.Vector Int -> DataFrame -> DataFrame -> T.Text -> DataFrame mkGroupedColumns indices df acc name = case (V.!) (columns df) (columnIndices df M.! name) of - Nothing -> error "Unexpected" - (Just (BoxedColumn column)) -> + BoxedColumn column -> let vs = indices `getIndices` column - in insertColumn name vs acc - (Just (OptionalColumn column)) -> + in insertVector name vs acc + OptionalColumn column -> let vs = indices `getIndices` column - in insertColumn name vs acc - (Just (UnboxedColumn column)) -> + in insertVector name vs acc + UnboxedColumn column -> let vs = indices `getIndicesUnboxed` column - in insertUnboxedColumn name vs acc + in insertUnboxedVector name vs acc groupColumns :: V.Vector (VU.Vector Int) -> DataFrame -> DataFrame -> T.Text -> DataFrame groupColumns indices df acc name = case (V.!) (columns df) (columnIndices df M.! name) of - Nothing -> df - (Just (BoxedColumn column)) -> + BoxedColumn column -> let vs = V.map (`getIndices` column) indices - in insertColumn' name (Just $ GroupedBoxedColumn vs) acc - (Just (OptionalColumn column)) -> + in insertColumn name (GroupedBoxedColumn vs) acc + OptionalColumn column -> let vs = V.map (`getIndices` column) indices - in insertColumn' name (Just $ GroupedBoxedColumn vs) acc - (Just (UnboxedColumn column)) -> + in insertColumn name (GroupedBoxedColumn vs) acc + UnboxedColumn column -> let vs = V.map (`getIndicesUnboxed` column) indices - in insertColumn' name (Just $ GroupedUnboxedColumn vs) acc - -data Aggregation = Count - | Mean - | Minimum - | Median - | Maximum - | Sum deriving (Show, Eq) - -groupByAgg :: Aggregation -> [T.Text] -> DataFrame -> DataFrame -groupByAgg agg columnNames df = let - in case agg of - Count -> insertColumnWithDefault @Int 1 (T.pack (show agg)) V.empty df - & groupBy columnNames - & reduceBy @Int VG.length "Count" - _ -> error "UNIMPLEMENTED" - --- O (k * n) Reduces a vector valued volumn with a given function. -reduceBy :: - forall a b . (Columnable a, Columnable b) => - (forall v . (VG.Vector v a) => v a -> b) -> - T.Text -> - DataFrame -> - DataFrame -reduceBy f name df = case getColumn name df of - Just ((GroupedBoxedColumn (column :: V.Vector (V.Vector a')))) -> case testEquality (typeRep @a) (typeRep @a') of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map f column)) df - Nothing -> error "Type error" - Just ((GroupedUnboxedColumn (column :: V.Vector (VU.Vector a')))) -> case testEquality (typeRep @a) (typeRep @a') of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map f column)) df - Nothing -> error "Type error" - _ -> error "Column is ungrouped" - -reduceByAgg :: Aggregation - -> T.Text - -> DataFrame - -> DataFrame -reduceByAgg agg name df = case agg of - Count -> case getColumn name df of - Just ((GroupedBoxedColumn (column :: V.Vector (V.Vector a')))) -> insertColumn' name (Just $ fromVector (VG.map VG.length column)) df - Just ((GroupedUnboxedColumn (column :: V.Vector (VU.Vector a')))) -> insertColumn' name (Just $ fromVector (VG.map VG.length column)) df - _ -> error $ "Cannot count ungrouped Column: " ++ T.unpack name - Mean -> case getColumn name df of - Just ((GroupedBoxedColumn (column :: V.Vector (V.Vector a')))) -> case testEquality (typeRep @a') (typeRep @Int) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map (SS.mean . VG.map fromIntegral) column)) df - Nothing -> case testEquality (typeRep @a') (typeRep @Double) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map SS.mean column)) df - Nothing -> case testEquality (typeRep @a') (typeRep @Float) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map (SS.mean . VG.map realToFrac) column)) df - Nothing -> error $ "Cannot get mean of non-numeric column: " ++ T.unpack name -- Not sure what to do with no numeric - return nothing??? - Just ((GroupedUnboxedColumn (column :: V.Vector (VU.Vector a')))) -> case testEquality (typeRep @a') (typeRep @Int) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map (SS.mean . VG.map fromIntegral) column)) df - Nothing -> case testEquality (typeRep @a') (typeRep @Double) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map SS.mean column)) df - Nothing -> case testEquality (typeRep @a') (typeRep @Float) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map (SS.mean . VG.map realToFrac) column)) df - Nothing -> error $ "Cannot get mean of non-numeric column: " ++ T.unpack name -- Not sure what to do with no numeric - return nothing??? - Minimum -> case getColumn name df of - Just ((GroupedBoxedColumn (column :: V.Vector (V.Vector a')))) -> insertColumn' name (Just $ fromVector (VG.map VG.minimum column)) df - Just ((GroupedUnboxedColumn (column :: V.Vector (VU.Vector a')))) -> insertColumn' name (Just $ fromVector (VG.map VG.minimum column)) df - Maximum -> case getColumn name df of - Just ((GroupedBoxedColumn (column :: V.Vector (V.Vector a')))) -> insertColumn' name (Just $ fromVector (VG.map VG.maximum column)) df - Just ((GroupedUnboxedColumn (column :: V.Vector (VU.Vector a')))) -> insertColumn' name (Just $ fromVector (VG.map VG.maximum column)) df - Sum -> case getColumn name df of - Just ((GroupedBoxedColumn (column :: V.Vector (V.Vector a')))) -> case testEquality (typeRep @a') (typeRep @Int) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map VG.sum column)) df - Nothing -> case testEquality (typeRep @a') (typeRep @Double) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map VG.sum column)) df - Nothing -> error $ "Cannot get sum of non-numeric column: " ++ T.unpack name -- Not sure what to do with no numeric - return nothing??? - Just ((GroupedUnboxedColumn (column :: V.Vector (VU.Vector a')))) -> case testEquality (typeRep @a') (typeRep @Int) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map VG.sum column)) df - Nothing -> case testEquality (typeRep @a') (typeRep @Double) of - Just Refl -> insertColumn' name (Just $ fromVector (VG.map VG.sum column)) df - Nothing -> error $ "Cannot get sum of non-numeric column: " ++ T.unpack name -- Not sure what to do with no numeric - return nothing??? - _ -> error "UNIMPLEMENTED" + in insertColumn name (GroupedUnboxedColumn vs) acc -aggregate :: [(T.Text, Aggregation)] -> DataFrame -> DataFrame +aggregate :: [(T.Text, UExpr)] -> DataFrame -> DataFrame aggregate aggs df = let - f (name, agg) d = cloneColumn name alias d & reduceByAgg agg alias - where alias = (T.pack . show) agg <> "_" <> name - in fold f aggs df & exclude (map fst aggs) - - -appendWithFrontMin :: (Ord a) => a -> [a] -> [a] -appendWithFrontMin x [] = [x] -appendWithFrontMin x xs@(f:rest) - | x < f = x:xs - | otherwise = f:x:rest -{-# INLINE appendWithFrontMin #-} + groupingColumns = Prelude.filter (\c -> not $ T.isPrefixOf "Grouped" (T.pack $ columnVersionString (fromMaybe (error "Unexpected") (getColumn c df)))) (columnNames df) + df' = select groupingColumns df + f (name, Wrap (expr :: Expr a)) d = let + value = interpret @a df expr + in insertColumn name (unwrapTypedColumn value) d + in fold f aggs df' distinct :: DataFrame -> DataFrame distinct df = groupBy (columnNames df) df diff --git a/src/DataFrame/Operations/Core.hs b/src/DataFrame/Operations/Core.hs --- a/src/DataFrame/Operations/Core.hs +++ b/src/DataFrame/Operations/Core.hs @@ -5,8 +5,6 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE Strict #-} module DataFrame.Operations.Core where import qualified Data.List as L @@ -41,7 +39,7 @@ {-# INLINE columnNames #-} -- | /O(n)/ Adds a vector to the dataframe. -insertColumn :: +insertVector :: forall a. Columnable a => -- | Column Name @@ -51,16 +49,16 @@ -- | DataFrame to add column to DataFrame -> DataFrame -insertColumn name xs = insertColumn' name (Just (fromVector xs)) -{-# INLINE insertColumn #-} +insertVector name xs = insertColumn name (fromVector xs) +{-# INLINE insertVector #-} cloneColumn :: T.Text -> T.Text -> DataFrame -> DataFrame cloneColumn original new df = fromMaybe (throw $ ColumnNotFoundException original "cloneColumn" (map fst $ M.toList $ columnIndices df)) $ do column <- getColumn original df - return $ insertColumn' new (Just column) df + return $ insertColumn new column df -- | /O(n)/ Adds an unboxed vector to the dataframe. -insertUnboxedColumn :: +insertUnboxedVector :: forall a. (Columnable a, VU.Unbox a) => -- | Column Name @@ -70,56 +68,29 @@ -- | DataFrame to add to column DataFrame -> DataFrame -insertUnboxedColumn name xs = insertColumn' name (Just (UnboxedColumn xs)) +insertUnboxedVector name xs = insertColumn name (UnboxedColumn xs) -- -- | /O(n)/ Add a column to the dataframe. Not meant for external use. -insertColumn' :: +insertColumn :: -- | Column Name T.Text -> -- | Column to add - Maybe Column -> + Column -> -- | DataFrame to add to column DataFrame -> DataFrame -insertColumn' _ Nothing d = d -insertColumn' name optCol@(Just column) d - | M.member name (columnIndices d) = let - i = (M.!) (columnIndices d) name - in d { columns = columns d V.// [(i, optCol)] } - | otherwise = insertNewColumn - where - l = columnLength column - (r, c) = dataframeDimensions d - diff = abs (l - r) - insertNewColumn - -- If we have a non-empty dataframe and we have more rows in the new column than the other column - -- we should make all the other columns have null and then add the new column. - | r > 0 && l > r = let - indexes = (map snd . L.sortBy (compare `on` snd). M.toList . columnIndices) d - nonEmptyColumns = L.foldl' (\acc i -> acc ++ [maybe (error "Unexpected") (expandColumn diff) (columns d V.! i)]) [] indexes - in fromNamedColumns (zip (columnNames d ++ [name]) (nonEmptyColumns ++ [column])) - | otherwise = let - (n:rest) = case freeIndices d of - [] -> [VG.length (columns d)..(VG.length (columns d) * 2 - 1)] - lst -> lst - columns' = if L.null (freeIndices d) - then columns d V.++ V.replicate (VG.length (columns d)) Nothing - else columns d - xs' - | diff <= 0 || null d = optCol - | otherwise = expandColumn diff <$> optCol - in d - { columns = columns' V.// [(n, xs')], - columnIndices = M.insert name n (columnIndices d), - freeIndices = rest, - dataframeDimensions = (max l r, c + 1) - } +insertColumn name column d = let + (r, c) = dataframeDimensions d + n = max (columnLength column) r + in case M.lookup name (columnIndices d) of + Just i -> DataFrame (V.map (expandColumn n) (columns d V.// [(i, column)])) (columnIndices d) (n, c) + Nothing -> DataFrame (V.map (expandColumn n) (columns d `V.snoc` column)) (M.insert name c (columnIndices d)) (n, c + 1) -- | /O(k)/ Add a column to the dataframe providing a default. -- This constructs a new vector and also may convert it -- to an unboxed vector if necessary. Since columns are usually -- large the runtime is dominated by the length of the list, k. -insertColumnWithDefault :: +insertVectorWithDefault :: forall a. (Columnable a) => -- | Default Value @@ -131,10 +102,10 @@ -- | DataFrame to add to column DataFrame -> DataFrame -insertColumnWithDefault defaultValue name xs d = +insertVectorWithDefault defaultValue name xs d = let (rows, _) = dataframeDimensions d values = xs V.++ V.replicate (rows - V.length xs) defaultValue - in insertColumn' name (Just $ fromVector values) d + in insertColumn name (fromVector values) d -- TODO: Add existence check in rename. rename :: T.Text -> T.Text -> DataFrame -> DataFrame @@ -160,31 +131,30 @@ -- | O(n) Returns the number of non-null columns in the dataframe and the type associated -- with each column. columnInfo :: DataFrame -> DataFrame -columnInfo df = empty & insertColumn' "Column Name" (Just $! fromList (map nameOfColumn infos)) - & insertColumn' "# Non-null Values" (Just $! fromList (map nonNullValues infos)) - & insertColumn' "# Null Values" (Just $! fromList (map nullValues infos)) - & insertColumn' "# Partially parsed" (Just $! fromList (map partiallyParsedValues infos)) - & insertColumn' "# Unique Values" (Just $! fromList (map uniqueValues infos)) - & insertColumn' "Type" (Just $! fromList (map typeOfColumn infos)) +columnInfo df = empty & insertColumn "Column Name" (fromList (map nameOfColumn infos)) + & insertColumn "# Non-null Values" (fromList (map nonNullValues infos)) + & insertColumn "# Null Values" (fromList (map nullValues infos)) + & insertColumn "# Partially parsed" (fromList (map partiallyParsedValues infos)) + & insertColumn "# Unique Values" (fromList (map uniqueValues infos)) + & insertColumn "Type" (fromList (map typeOfColumn infos)) where infos = L.sortBy (compare `on` nonNullValues) (V.ifoldl' go [] (columns df)) :: [ColumnInfo] indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df)) columnName i = M.lookup i indexMap - go acc i Nothing = acc - go acc i (Just col@(OptionalColumn (c :: V.Vector a))) = let + go acc i col@(OptionalColumn (c :: V.Vector a)) = let cname = columnName i countNulls = nulls col countPartial = partiallyParsed col columnType = T.pack $ show $ typeRep @a unique = S.size $ VG.foldr S.insert S.empty c in if isNothing cname then acc else ColumnInfo (fromMaybe "" cname) (columnLength col - countNulls) countNulls countPartial unique columnType : acc - go acc i (Just col@(BoxedColumn (c :: V.Vector a))) = let + go acc i col@(BoxedColumn (c :: V.Vector a)) = let cname = columnName i countPartial = partiallyParsed col columnType = T.pack $ show $ typeRep @a unique = S.size $ VG.foldr S.insert S.empty c in if isNothing cname then acc else ColumnInfo (fromMaybe "" cname) (columnLength col) 0 countPartial unique columnType : acc - go acc i (Just col@(UnboxedColumn c)) = let + go acc i col@(UnboxedColumn c) = let cname = columnName i columnType = T.pack $ columnTypeString col unique = S.size $ VG.foldr S.insert S.empty c @@ -215,10 +185,10 @@ partiallyParsed _ = 0 fromNamedColumns :: [(T.Text, Column)] -> DataFrame -fromNamedColumns = L.foldl' (\df (!name, !column) -> insertColumn' name (Just $! column) df) empty +fromNamedColumns = L.foldl' (\df (name, column) -> insertColumn name column df) empty -fromUnamedColumns :: [Column] -> DataFrame -fromUnamedColumns = fromNamedColumns . zip (map (T.pack . show) [0..]) +fromUnnamedColumns :: [Column] -> DataFrame +fromUnnamedColumns = fromNamedColumns . zip (map (T.pack . show) [0..]) -- | O (k * n) Counts the occurences of each value in a given column. valueCounts :: forall a. (Columnable a) => T.Text -> DataFrame -> [(a, Int)] diff --git a/src/DataFrame/Operations/Join.hs b/src/DataFrame/Operations/Join.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Operations/Join.hs @@ -0,0 +1,58 @@ +{-# LANGUAGE OverloadedStrings #-} +module DataFrame.Operations.Join where + +import qualified Data.Map.Strict as M +import qualified Data.Vector as VB +import qualified Data.Vector.Unboxed as VU +import qualified Data.Text as T +import DataFrame.Internal.Column as D +import DataFrame.Internal.DataFrame as D +import DataFrame.Operations.Aggregation as D +import DataFrame.Operations.Core as D + +data JoinType = INNER + | LEFT + | RIGHT + | FULL_OUTER + +join :: JoinType + -> [T.Text] + -> DataFrame -- Right hand side + -> DataFrame -- Left hand side + -> DataFrame +join INNER xs right = innerJoin xs right +join LEFT xs right = error "UNIMPLEMENTED" +join RIGHT xs right = error "UNIMPLEMENTED" +join FULL_OUTER xs right = error "UNIMPLEMENTED" + +-- Create row representation for each of the two dataframes +-- get the product of left and right counts for each key +innerJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame +innerJoin cs right left = let + leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) + leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) + -- key -> [index0, index1] + leftKeyCountsAndIndices = VU.foldr (\(i, v) acc -> M.insertWith (++) v [i] acc) M.empty (VU.indexed leftRowRepresentations) + -- key -> [index0, index1] + rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right) + rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) + rightKeyCountsAndIndices = VU.foldr (\(i, v) acc -> M.insertWith (++) v [i] acc) M.empty (VU.indexed rightRowRepresentations) + -- key -> [(left_indexes0, right_indexes1)] + mergedKeyCountsAndIndices = M.foldrWithKey (\k v m -> if k `M.member` rightKeyCountsAndIndices then M.insert k (VU.fromList v, VU.fromList (rightKeyCountsAndIndices M.! k)) m else m) M.empty leftKeyCountsAndIndices + -- [(ints, ints)] + leftAndRightIndicies = M.elems mergedKeyCountsAndIndices + -- [(ints, ints)] (expanded to n * m) + expandedIndices = map (\(l, r) -> (mconcat (replicate (VU.length r) l), mconcat (replicate (VU.length l) r))) leftAndRightIndicies + expandedLeftIndicies = mconcat (map fst expandedIndices) + expandedRightIndicies = mconcat (map snd expandedIndices) + -- df + expandedLeft = left { columns = VB.map (D.atIndicesStable expandedLeftIndicies) (D.columns left), dataframeDimensions = (VU.length expandedLeftIndicies, snd (D.dataframeDimensions left))} + -- df + expandedRight = right { columns = VB.map (D.atIndicesStable expandedRightIndicies) (D.columns right), dataframeDimensions = (VU.length expandedRightIndicies, snd (D.dataframeDimensions right))} + -- [string] + leftColumns = D.columnNames left + rightColumns = D.columnNames right + initDf = expandedLeft + insertIfPresent _ Nothing df = df + insertIfPresent name (Just c) df = D.insertColumn name c df + in D.fold (\name df -> if name `elem` cs then df else (if name `elem` leftColumns then insertIfPresent ("Right_" <> name) (D.getColumn name expandedRight) df else insertIfPresent name (D.getColumn name expandedRight) df)) rightColumns initDf diff --git a/src/DataFrame/Operations/Merge.hs b/src/DataFrame/Operations/Merge.hs --- a/src/DataFrame/Operations/Merge.hs +++ b/src/DataFrame/Operations/Merge.hs @@ -1,5 +1,4 @@ {-# LANGUAGE InstanceSigs #-} -{-# LANGUAGE Strict #-} module DataFrame.Operations.Merge where import qualified Data.List as L @@ -9,6 +8,8 @@ import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Operations.Core as D +import Data.Maybe + instance Semigroup D.DataFrame where (<>) :: D.DataFrame -> D.DataFrame -> D.DataFrame (<>) a b = let @@ -16,8 +17,12 @@ columnsInA = D.columnNames a addColumns a' b' df name | fst (D.dimensions a') == 0 && fst (D.dimensions b') == 0 = df - | fst (D.dimensions a') == 0 = D.insertColumn' name (D.getColumn name b') df - | fst (D.dimensions b') == 0 = D.insertColumn' name (D.getColumn name a') df + | fst (D.dimensions a') == 0 = fromMaybe df $ do + col <- D.getColumn name b' + pure $ D.insertColumn name col df + | fst (D.dimensions b') == 0 = fromMaybe df $ do + col <- D.getColumn name a' + pure $ D.insertColumn name col df | otherwise = let numColumnsA = (fst $ D.dimensions a') numColumnsB = (fst $ D.dimensions b') @@ -26,11 +31,13 @@ optB = D.getColumn name b' in case optB of Nothing -> case optA of - Nothing -> D.insertColumn' name (Just (D.fromList ([] :: [T.Text]))) df - Just a'' -> D.insertColumn' name (Just (D.expandColumn numColumnsB a'')) df + Nothing -> D.insertColumn name (D.fromList ([] :: [T.Text])) df + Just a'' -> D.insertColumn name (D.expandColumn numColumnsB a'') df Just b'' -> case optA of - Nothing -> D.insertColumn' name (Just (D.leftExpandColumn numColumnsA b'')) df - Just a'' -> D.insertColumn' name (D.concatColumns a'' b'') df + Nothing -> D.insertColumn name (D.leftExpandColumn numColumnsA b'') df + Just a'' -> fromMaybe df $ do + concatedColumns <- D.concatColumns a'' b'' + pure $ D.insertColumn name concatedColumns df in L.foldl' (addColumns a b) D.empty (D.columnNames a `L.union` D.columnNames b) instance Monoid D.DataFrame where diff --git a/src/DataFrame/Operations/Sorting.hs b/src/DataFrame/Operations/Sorting.hs --- a/src/DataFrame/Operations/Sorting.hs +++ b/src/DataFrame/Operations/Sorting.hs @@ -29,5 +29,5 @@ -- TODO: Remove the SortOrder defintion from operations so we can share it between here and internal and -- we don't have to do this Bool mapping. indexes = sortedIndexes' (order == Ascending) (toRowVector names df) - pick idxs col = atIndicesStable idxs <$> col + pick idxs col = atIndicesStable idxs col in df {columns = V.map (pick indexes) (columns df)} diff --git a/src/DataFrame/Operations/Statistics.hs b/src/DataFrame/Operations/Statistics.hs --- a/src/DataFrame/Operations/Statistics.hs +++ b/src/DataFrame/Operations/Statistics.hs @@ -4,8 +4,8 @@ {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE StrictData #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE BangPatterns #-} module DataFrame.Operations.Statistics where import qualified Data.List as L @@ -43,8 +43,8 @@ Nothing -> case testEquality (typeRep @a) (typeRep @String) of Just Refl -> T.pack c' Nothing -> (T.pack . show) c' - initDf = empty & insertColumn "Statistic" (V.fromList ["Count" :: T.Text, "Percentage (%)"]) - in L.foldl' (\df (col, k) -> insertColumn (vText col) (V.fromList [k, k * 100 `div` total]) df) initDf counts + initDf = empty & insertVector "Statistic" (V.fromList ["Count" :: T.Text, "Percentage (%)"]) + in L.foldl' (\df (col, k) -> insertVector (vText col) (V.fromList [k, k * 100 `div` total]) df) initDf counts Just ((OptionalColumn (column :: V.Vector a))) -> let counts = valueCounts @a name df total = P.sum $ map snd counts @@ -54,8 +54,8 @@ Nothing -> case testEquality (typeRep @a) (typeRep @String) of Just Refl -> T.pack c' Nothing -> (T.pack . show) c' - initDf = empty & insertColumn "Statistic" (V.fromList ["Count" :: T.Text, "Percentage (%)"]) - in L.foldl' (\df (col, k) -> insertColumn (vText col) (V.fromList [k, k * 100 `div` total]) df) initDf counts + initDf = empty & insertVector "Statistic" (V.fromList ["Count" :: T.Text, "Percentage (%)"]) + in L.foldl' (\df (col, k) -> insertVector (vText col) (V.fromList [k, k * 100 `div` total]) df) initDf counts Just ((UnboxedColumn (column :: VU.Vector a))) -> let counts = valueCounts @a name df total = P.sum $ map snd counts @@ -65,11 +65,12 @@ Nothing -> case testEquality (typeRep @a) (typeRep @String) of Just Refl -> T.pack c' Nothing -> (T.pack . show) c' - initDf = empty & insertColumn "Statistic" (V.fromList ["Count" :: T.Text, "Percentage (%)"]) - in L.foldl' (\df (col, k) -> insertColumn (vText col) (V.fromList [k, k * 100 `div` total]) df) initDf counts + initDf = empty & insertVector "Statistic" (V.fromList ["Count" :: T.Text, "Percentage (%)"]) + in L.foldl' (\df (col, k) -> insertVector (vText col) (V.fromList [k, k * 100 `div` total]) df) initDf counts + _ -> error $ "There are ungrouped columns" mean :: T.Text -> DataFrame -> Maybe Double -mean = applyStatistic SS.mean +mean = applyStatistic mean' median :: T.Text -> DataFrame -> Maybe Double median = applyStatistic (SS.median SS.medianUnbiased) @@ -81,7 +82,7 @@ skewness = applyStatistic SS.skewness variance :: T.Text -> DataFrame -> Maybe Double -variance = applyStatistic SS.variance +variance = applyStatistic variance' interQuartileRange :: T.Text -> DataFrame -> Maybe Double interQuartileRange = applyStatistic (SS.midspread SS.medianUnbiased 4) @@ -101,26 +102,25 @@ Nothing -> Nothing _ -> Nothing -sum :: T.Text -> DataFrame -> Maybe Double +sum :: forall a. (Columnable a, Num a, VU.Unbox a) => T.Text -> DataFrame -> Maybe a sum name df = case getColumn name df of Nothing -> throw $ ColumnNotFoundException name "sum" (map fst $ M.toList $ columnIndices df) - Just ((UnboxedColumn (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @Int) of - Just Refl -> Just $ VG.sum (VU.map fromIntegral column) - Nothing -> case testEquality (typeRep @a') (typeRep @Double) of - Just Refl -> Just $ VG.sum column - Nothing -> Nothing + Just ((UnboxedColumn (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of + Just Refl -> Just $ VG.sum column + Nothing -> Nothing applyStatistic :: (VU.Vector Double -> Double) -> T.Text -> DataFrame -> Maybe Double -applyStatistic f name df = do - column <- getColumn name df - if columnTypeString column == "Double" - then reduceColumn f column - else do - matching <- asum [mapColumn (fromIntegral :: Int -> Double) column, - mapColumn (fromIntegral :: Integer -> Double) column, - mapColumn (realToFrac :: Float -> Double) column, - Just column ] - reduceColumn f matching +applyStatistic f name df = case getColumn name df of + Nothing -> throw $ ColumnNotFoundException name "applyStatistic" (map fst $ M.toList $ columnIndices df) + Just column@(UnboxedColumn (col :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of + Just Refl -> reduceColumn f column + Nothing -> do + matching <- asum [mapColumn (fromIntegral :: Int -> Double) column, + mapColumn (fromIntegral :: Integer -> Double) column, + mapColumn (realToFrac :: Float -> Double) column, + Just column ] + reduceColumn f matching + _ -> Nothing applyStatistics :: (VU.Vector Double -> VU.Vector Double) -> T.Text -> DataFrame -> Maybe (VU.Vector Double) applyStatistics f name df = case getColumn name df of @@ -135,7 +135,7 @@ summarize :: DataFrame -> DataFrame summarize df = fold columnStats (columnNames df) (fromNamedColumns [("Statistic", fromList ["Mean" :: T.Text, "Minimum", "25%" ,"Median", "75%", "Max", "StdDev", "IQR", "Skewness"])]) - where columnStats name d = if all isJust (stats name) then insertUnboxedColumn name (VU.fromList (map (roundTo 2 . fromMaybe 0) $ stats name)) d else d + where columnStats name d = if all isJust (stats name) then insertUnboxedVector name (VU.fromList (map (roundTo 2 . fromMaybe 0) $ stats name)) d else d stats name = let quantiles = applyStatistics (SS.quantilesVec SS.medianUnbiased (VU.fromList [0,1,2,3,4]) 4) name df min' = flip (VG.!) 0 <$> quantiles @@ -155,3 +155,27 @@ skewness name df] roundTo :: Int -> Double -> Double roundTo n x = fromInteger (round $ x * (10^n)) / (10.0^^n) + +mean' :: VU.Vector Double -> Double +mean' samp = let + (!total, !n) = VG.foldl' (\(!total, !n) v -> (total + v, n + 1)) (0 :: Double, 0 :: Int) samp + in total / fromIntegral n + +-- accumulator: count, mean, m2 +data VarAcc = VarAcc !Int !Double !Double deriving Show + +step :: VarAcc -> Double -> VarAcc +step (VarAcc !n !mean !m2) !x = + let !n' = n + 1 + !delta = x - mean + !mean' = mean + delta / fromIntegral n' + !m2' = m2 + delta * (x - mean') + in VarAcc n' mean' m2' + +computeVariance :: VarAcc -> Double +computeVariance (VarAcc n _ m2) + | n < 2 = 0 -- or error "variance of <2 samples" + | otherwise = m2 / fromIntegral (n - 1) + +variance' :: VU.Vector Double -> Double +variance' = computeVariance . VG.foldl' step (VarAcc 0 0 0) diff --git a/src/DataFrame/Operations/Subset.hs b/src/DataFrame/Operations/Subset.hs --- a/src/DataFrame/Operations/Subset.hs +++ b/src/DataFrame/Operations/Subset.hs @@ -1,4 +1,3 @@ -{-# LANGUAGE BangPatterns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -31,32 +30,32 @@ -- | O(k * n) Take the first n rows of a DataFrame. take :: Int -> DataFrame -> DataFrame -take n d = d {columns = V.map (takeColumn n' <$>) (columns d), dataframeDimensions = (n', c)} +take n d = d {columns = V.map (takeColumn n') (columns d), dataframeDimensions = (n', c)} where (r, c) = dataframeDimensions d n' = clip n 0 r takeLast :: Int -> DataFrame -> DataFrame -takeLast n d = d {columns = V.map (takeLastColumn n' <$>) (columns d), dataframeDimensions = (n', c)} +takeLast n d = d {columns = V.map (takeLastColumn n') (columns d), dataframeDimensions = (n', c)} where (r, c) = dataframeDimensions d n' = clip n 0 r drop :: Int -> DataFrame -> DataFrame -drop n d = d {columns = V.map (sliceColumn n' (max (r - n') 0) <$>) (columns d), dataframeDimensions = (max (r - n') 0, c)} +drop n d = d {columns = V.map (sliceColumn n' (max (r - n') 0)) (columns d), dataframeDimensions = (max (r - n') 0, c)} where (r, c) = dataframeDimensions d n' = clip n 0 r dropLast :: Int -> DataFrame -> DataFrame -dropLast n d = d {columns = V.map (sliceColumn 0 n' <$>) (columns d), dataframeDimensions = (n', c)} +dropLast n d = d {columns = V.map (sliceColumn 0 n') (columns d), dataframeDimensions = (n', c)} where (r, c) = dataframeDimensions d n' = clip (r - n) 0 r -- | O(k * n) Take a range of rows of a DataFrame. range :: (Int, Int) -> DataFrame -> DataFrame -range (start, end) d = d {columns = V.map (sliceColumn (clip start 0 r) n' <$>) (columns d), dataframeDimensions = (n', c)} +range (start, end) d = d {columns = V.map (sliceColumn (clip start 0 r) n') (columns d), dataframeDimensions = (n', c)} where (r, c) = dataframeDimensions d n' = clip (end - start) 0 r @@ -79,7 +78,7 @@ DataFrame filter filterColumnName condition df = case getColumn filterColumnName df of Nothing -> throw $ ColumnNotFoundException filterColumnName "filter" (map fst $ M.toList $ columnIndices df) - Just column -> case ifoldlColumn (\s i v -> if condition v then S.insert i s else s) S.empty column of + Just column -> case findIndices condition column of Nothing -> throw $ TypeMismatchException (MkTypeErrorContext { userType = Right $ typeRep @a , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) @@ -87,8 +86,8 @@ , callingFunctionName = Just "filter"}) Just indexes -> let c' = snd $ dataframeDimensions df - pick idxs col = atIndices idxs <$> col - in df {columns = V.map (pick indexes) (columns df), dataframeDimensions = (S.size indexes, c')} + pick idxs col = atIndicesStable idxs col + in df {columns = V.map (pick indexes) (columns df), dataframeDimensions = (VG.length indexes, c')} -- | O(k) a version of filter where the predicate comes first. -- @@ -103,9 +102,9 @@ filterWhere :: Expr Bool -> DataFrame -> DataFrame filterWhere expr df = let (TColumn col) = interpret @Bool df expr - (Just indexes) = VU.convert . V.map (fromMaybe 0) . V.filter isJust . toVector @(Maybe Int) <$> imapColumn (\i satisfied -> if satisfied then Just i else Nothing) col + (Just indexes) = findIndices (==True) col c' = snd $ dataframeDimensions df - pick idxs col = atIndicesStable idxs <$> col + pick idxs col = atIndicesStable idxs col in df {columns = V.map (pick indexes) (columns df), dataframeDimensions = (VU.length indexes, c')} @@ -143,18 +142,9 @@ | any (`notElem` columnNames df) cs = throw $ ColumnNotFoundException (T.pack $ show $ cs L.\\ columnNames df) "select" (columnNames df) | otherwise = L.foldl' addKeyValue empty cs where - cIndexAssoc = M.toList $ columnIndices df - remaining = L.filter (\(!c, _) -> c `elem` cs) cIndexAssoc - removed = cIndexAssoc L.\\ remaining - indexes = map snd remaining - (r, c) = dataframeDimensions df - addKeyValue d k = - d - { columns = V.imap (\i v -> if i `notElem` indexes then Nothing else v) (columns df), - columnIndices = M.fromList remaining, - freeIndices = map snd removed ++ freeIndices df, - dataframeDimensions = (r, L.length remaining) - } + addKeyValue d k = fromMaybe df $ do + col <- getColumn k df + pure $ insertColumn k col d -- | O(n) select columns by index range of column names. selectIntRange :: (Int, Int) -> DataFrame -> DataFrame diff --git a/src/DataFrame/Operations/Transformations.hs b/src/DataFrame/Operations/Transformations.hs --- a/src/DataFrame/Operations/Transformations.hs +++ b/src/DataFrame/Operations/Transformations.hs @@ -3,8 +3,6 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE Strict #-} -{-# LANGUAGE StrictData #-} module DataFrame.Operations.Transformations where import qualified Data.List as L @@ -59,14 +57,14 @@ , errorColumnName = Just (T.unpack columnName) , callingFunctionName = Just "apply" }) - column' -> Right $ insertColumn' columnName column' d + Just column' -> Right $ insertColumn columnName column' d -- | O(k) Apply a function to a combination of columns in a dataframe and -- add the result into `alias` column. derive :: forall a . Columnable a => T.Text -> Expr a -> DataFrame -> DataFrame derive name expr df = let value = interpret @a df expr - in insertColumn' name (Just (unwrapTypedColumn value)) df + in insertColumn name (unwrapTypedColumn value) df -- | O(k * n) Apply a function to given column names in a dataframe. @@ -150,7 +148,7 @@ , errorColumnName = Just (T.unpack columnName) , callingFunctionName = Just "applyAtIndex" }) - column' -> insertColumn' columnName column' df + Just column' -> insertColumn columnName column' df impute :: forall b . diff --git a/src/DataFrame/Operations/Typing.hs b/src/DataFrame/Operations/Typing.hs --- a/src/DataFrame/Operations/Typing.hs +++ b/src/DataFrame/Operations/Typing.hs @@ -22,9 +22,8 @@ parseDefaults :: Bool -> DataFrame -> DataFrame parseDefaults safeRead df = df {columns = V.map (parseDefault safeRead) (columns df)} -parseDefault :: Bool -> Maybe Column -> Maybe Column -parseDefault _ Nothing = Nothing -parseDefault safeRead (Just (BoxedColumn (c :: V.Vector a))) = let +parseDefault :: Bool -> Column -> Column +parseDefault safeRead (BoxedColumn (c :: V.Vector a)) = let parseTimeOpt s = parseTimeM {- Accept leading/trailing whitespace -} True defaultTimeLocale "%Y-%m-%d" (T.unpack s) :: Maybe Day unsafeParseTime s = parseTimeOrError {- Accept leading/trailing whitespace -} True defaultTimeLocale "%Y-%m-%d" (T.unpack s) :: Day in case (typeRep @a) `testEquality` (typeRep @T.Text) of @@ -33,8 +32,8 @@ emptyToNothing v = if isNullish (T.pack v) then Nothing else Just v safeVector = V.map emptyToNothing c hasNulls = V.foldl' (\acc v -> if isNothing v then acc || True else acc) False safeVector - in Just $ if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c - Nothing -> Just $ BoxedColumn c + in if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c + Nothing -> BoxedColumn c Just Refl -> let example = T.strip (V.head c) emptyToNothing v = if isNullish v then Nothing else Just v @@ -42,12 +41,12 @@ Just _ -> let safeVector = V.map ((=<<) readInt . emptyToNothing) c hasNulls = V.elem Nothing safeVector - in Just $ if safeRead && hasNulls then BoxedColumn safeVector else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!))) + in if safeRead && hasNulls then BoxedColumn safeVector else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!))) Nothing -> case readDouble example of Just _ -> let safeVector = V.map ((=<<) readDouble . emptyToNothing) c hasNulls = V.elem Nothing safeVector - in Just $ if safeRead && hasNulls then BoxedColumn safeVector else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!))) + in if safeRead && hasNulls then BoxedColumn safeVector else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!))) Nothing -> case parseTimeOpt example of Just d -> let -- failed parse should be Either, nullish should be Maybe @@ -60,7 +59,7 @@ toMaybe (Right value) = Just value lefts = V.filter isLeft safeVector onlyNulls = (not (V.null lefts) && V.all (isNullish . fromLeft "non-null") lefts) - in Just $ if safeRead + in if safeRead then if onlyNulls then BoxedColumn (V.map toMaybe safeVector) else if V.any isLeft safeVector @@ -70,5 +69,5 @@ Nothing -> let safeVector = V.map emptyToNothing c hasNulls = V.any isNullish c - in Just $ if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c + in if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c parseDefault safeRead column = column diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -44,20 +44,20 @@ -- parsing. parseDate :: Test parseDate = let - expected = Just $ DI.BoxedColumn (V.fromList [fromGregorian 2020 02 14, fromGregorian 2021 02 14, fromGregorian 2022 02 14]) - actual = D.parseDefault True $ Just $ DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-14", "2022-02-14"]) + expected = DI.BoxedColumn (V.fromList [fromGregorian 2020 02 14, fromGregorian 2021 02 14, fromGregorian 2022 02 14]) + actual = D.parseDefault True (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-14", "2022-02-14"])) in TestCase (assertEqual "Correctly parses gregorian date" expected actual) incompleteDataParseEither :: Test incompleteDataParseEither = let - expected = Just $ DI.BoxedColumn (V.fromList [Right $ fromGregorian 2020 02 14, Left ("2021-02-" :: T.Text), Right $ fromGregorian 2022 02 14]) - actual = D.parseDefault True $ Just $ DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-", "2022-02-14"]) + expected = DI.BoxedColumn (V.fromList [Right $ fromGregorian 2020 02 14, Left ("2021-02-" :: T.Text), Right $ fromGregorian 2022 02 14]) + actual = D.parseDefault True (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-", "2022-02-14"])) in TestCase (assertEqual "Parses Either for gregorian date" expected actual) incompleteDataParseMaybe :: Test incompleteDataParseMaybe = let - expected = Just $ DI.BoxedColumn (V.fromList [Just $ fromGregorian 2020 02 14, Nothing, Just $ fromGregorian 2022 02 14]) - actual = D.parseDefault True $ Just $ DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "", "2022-02-14"]) + expected = DI.BoxedColumn (V.fromList [Just $ fromGregorian 2020 02 14, Nothing, Just $ fromGregorian 2022 02 14]) + actual = D.parseDefault True (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "", "2022-02-14"])) in TestCase (assertEqual "Parses Maybe for gregorian date with null/empty" expected actual) parseTests :: [Test] diff --git a/tests/Operations/Derive.hs b/tests/Operations/Derive.hs --- a/tests/Operations/Derive.hs +++ b/tests/Operations/Derive.hs @@ -4,6 +4,7 @@ module Operations.Derive where import qualified DataFrame as D +import qualified DataFrame.Functions as F import qualified DataFrame as DI import qualified DataFrame as DE import qualified Data.Text as T @@ -27,7 +28,7 @@ deriveWAI = TestCase (assertEqual "derive works with column expression" (Just $ DI.BoxedColumn (V.fromList (zipWith (\n c -> show n ++ [c]) [1..26] ['a'..'z']))) (DI.getColumn "test4" $ D.derive "test4" ( - D.lift2 (++) (D.lift show (D.col @Int "test1")) (D.lift (: ([] :: [Char])) (D.col @Char "test3")) + F.lift2 (++) (F.lift show (F.col @Int "test1")) (F.lift (: ([] :: [Char])) (F.col @Char "test3")) ) testData)) tests :: [Test] diff --git a/tests/Operations/InsertColumn.hs b/tests/Operations/InsertColumn.hs --- a/tests/Operations/InsertColumn.hs +++ b/tests/Operations/InsertColumn.hs @@ -13,64 +13,64 @@ testData :: D.DataFrame testData = D.fromNamedColumns [ ("test1", DI.fromList([1..26] :: [Int])) - , ("test2", DI.fromList['a'..'z']) - , ("test3", DI.fromList([1..26] :: [Int])) - , ("test4", DI.fromList['a'..'z']) - , ("test5", DI.fromList([1..26] :: [Int])) - , ("test6", DI.fromList['a'..'z']) - , ("test7", DI.fromList([1..26] :: [Int])) - , ("test8", DI.fromList['a'..'z']) + , ("test2", DI.fromList ['a'..'z']) + , ("test3", DI.fromList ([1..26] :: [Int])) + , ("test4", DI.fromList ['a'..'z']) + , ("test5", DI.fromList ([1..26] :: [Int])) + , ("test6", DI.fromList ['a'..'z']) + , ("test7", DI.fromList ([1..26] :: [Int])) + , ("test8", DI.fromList ['a'..'z']) ] -- Adding a boxed vector to an empty dataframe creates a new column boxed containing the vector elements. addBoxedColumn :: Test addBoxedColumn = TestCase (assertEqual "Two columns should be equal" (Just $ DI.BoxedColumn (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"])) - (DI.getColumn "new" $ D.insertColumn "new" (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"]) D.empty)) + (DI.getColumn "new" $ D.insertVector "new" (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"]) D.empty)) addBoxedColumn' :: Test addBoxedColumn' = TestCase (assertEqual "Two columns should be equal" (Just $ DI.fromList["Thuba" :: T.Text, "Zodwa", "Themba"]) - (DI.getColumn "new" $ D.insertColumn' "new" (Just $ DI.fromList["Thuba" :: T.Text, "Zodwa", "Themba"]) D.empty)) + (DI.getColumn "new" $ D.insertColumn "new" (DI.fromList["Thuba" :: T.Text, "Zodwa", "Themba"]) D.empty)) -- Adding an boxed vector with an unboxable type (Int/Double) to an empty dataframe creates a new column boxed containing the vector elements. addUnboxedColumn :: Test addUnboxedColumn = TestCase (assertEqual "Value should be boxed" (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 2, 3])) - (DI.getColumn "new" $ D.insertColumn "new" (V.fromList [1 :: Int, 2, 3]) D.empty)) + (DI.getColumn "new" $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3]) D.empty)) addUnboxedColumn' :: Test addUnboxedColumn' = TestCase (assertEqual "Value should be boxed" (Just $ DI.fromList[1 :: Int, 2, 3]) - (DI.getColumn "new" $ D.insertColumn' "new" (Just $ DI.fromList[1 :: Int, 2, 3]) D.empty)) + (DI.getColumn "new" $ D.insertColumn "new" (DI.fromList[1 :: Int, 2, 3]) D.empty)) -- Adding a column with less values than the current DF dimensions adds column with optionals. addSmallerColumnBoxed :: Test addSmallerColumnBoxed = TestCase ( assertEqual "Missing values should be replaced with Nothing" (Just $ DI.OptionalColumn (V.fromList [Just "a" :: Maybe T.Text, Just "b", Just "c", Nothing, Nothing])) - (DI.getColumn "newer" $ D.insertColumn "newer" (V.fromList ["a" :: T.Text, "b", "c"]) $ D.insertColumn "new" (V.fromList ["a" :: T.Text, "b", "c", "d", "e"]) D.empty) + (DI.getColumn "newer" $ D.insertVector "newer" (V.fromList ["a" :: T.Text, "b", "c"]) $ D.insertVector "new" (V.fromList ["a" :: T.Text, "b", "c", "d", "e"]) D.empty) ) addSmallerColumnUnboxed :: Test addSmallerColumnUnboxed = TestCase ( assertEqual "Missing values should be replaced with Nothing" (Just $ DI.OptionalColumn (V.fromList [Just 1 :: Maybe Int, Just 2, Just 3, Nothing, Nothing])) - (DI.getColumn "newer" $ D.insertColumn "newer" (V.fromList [1 :: Int, 2, 3]) $ D.insertColumn "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty) + (DI.getColumn "newer" $ D.insertVector "newer" (V.fromList [1 :: Int, 2, 3]) $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty) ) insertColumnWithDefaultFillsWithDefault :: Test insertColumnWithDefaultFillsWithDefault = TestCase ( assertEqual "Missing values should be replaced with Nothing" (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 2, 3, 0, 0])) - (DI.getColumn "newer" $ D.insertColumnWithDefault 0 "newer" (V.fromList [1 :: Int, 2, 3]) $ D.insertColumn "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty) + (DI.getColumn "newer" $ D.insertVectorWithDefault 0 "newer" (V.fromList [1 :: Int, 2, 3]) $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty) ) insertColumnWithDefaultFillsLargerNoop :: Test insertColumnWithDefaultFillsLargerNoop = TestCase ( assertEqual "Lists should be the same size" (Just $ DI.UnboxedColumn (VU.fromList [(6 :: Int)..10])) - (DI.getColumn "newer" $ D.insertColumnWithDefault 0 "newer" (V.fromList [(6 :: Int)..10]) $ D.insertColumn "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty) + (DI.getColumn "newer" $ D.insertVectorWithDefault 0 "newer" (V.fromList [(6 :: Int)..10]) $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty) ) addLargerColumnBoxed :: Test @@ -78,28 +78,29 @@ TestCase (assertEqual "Smaller lists should grow and contain optionals" (D.fromNamedColumns [("new", D.fromList [Just "a" :: Maybe T.Text, Just "b", Just "c", Nothing, Nothing]), ("newer", D.fromList ["a" :: T.Text, "b", "c", "d", "e"])]) - (D.insertColumn "newer" (V.fromList ["a" :: T.Text, "b", "c", "d", "e"]) - $ D.insertColumn "new" (V.fromList ["a" :: T.Text, "b", "c"]) D.empty)) + (D.insertVector "newer" (V.fromList ["a" :: T.Text, "b", "c", "d", "e"]) + $ D.insertVector "new" (V.fromList ["a" :: T.Text, "b", "c"]) D.empty)) + addLargerColumnUnboxed :: Test addLargerColumnUnboxed = TestCase (assertEqual "Smaller lists should grow and contain optionals" (D.fromNamedColumns [("old", D.fromList [Just 1 :: Maybe Int, Just 2, Nothing, Nothing, Nothing]), ("new", D.fromList [Just 1 :: Maybe Int, Just 2, Just 3, Nothing, Nothing]), ("newer", D.fromList [1 :: Int, 2, 3, 4, 5])]) - (D.insertColumn "newer" (V.fromList [1 :: Int, 2, 3, 4, 5]) - $ D.insertColumn "new" (V.fromList [1 :: Int, 2, 3]) $ - D.insertColumn "old" (V.fromList [1 :: Int, 2]) D.empty)) + (D.insertVector "newer" (V.fromList [1 :: Int, 2, 3, 4, 5]) + $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3]) $ + D.insertVector "old" (V.fromList [1 :: Int, 2]) D.empty)) dimensionsChangeAfterAdd :: Test dimensionsChangeAfterAdd = TestCase (assertEqual "should be (26, 3)" (26, 9) - (D.dimensions $ D.insertColumn @Int "new" (V.fromList [1..26]) testData)) + (D.dimensions $ D.insertVector @Int "new" (V.fromList [1..26]) testData)) dimensionsNotChangedAfterDuplicate :: Test dimensionsNotChangedAfterDuplicate = TestCase (assertEqual "should be (26, 3)" (26, 9) - (D.dimensions $ D.insertColumn @Int "new" (V.fromList [1..26]) - $ D.insertColumn @Int "new" (V.fromList [1..26]) testData)) + (D.dimensions $ D.insertVector @Int "new" (V.fromList [1..26]) + $ D.insertVector @Int "new" (V.fromList [1..26]) testData)) tests :: [Test]