diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Revision history for dataframe +## 0.3.3.7 +* Many functions how rely on expressions (not strings). +* full, left, and right join now implemented. +* fastCsv now strips quotations from text. +* Add "NA" as a nullish pattern. +* Add bin parameter to terminal plotting. +* Implement filterAllNothing for null handling. +* Remove behaviour where we parse mixed types as `Either` +* Add `whenPresent`, `whenBothPresent` and `recode` functions. +* Web charts now show on first load. +* Add deriveMay function for multiple column derivations. + ## 0.3.3.6 * Fix bug where doubles were parsing as ints * Fix bugs where optionals were left in boxed column (instead of optionals) diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ hackage Latest Release - + C/I @@ -31,4 +31,66 @@ * Designed for interactivity: expressive syntax, helpful error messages, and sensible defaults. * Works seamlessly in both command-line and notebook environments—great for exploration and scripting alike. -For an installation guide and tutorials checkout the [project documentation](https://dataframe.readthedocs.io/) and for an API reference checkout the [hackage documentation](https://hackage-content.haskell.org/package/dataframe-0.3.3.6/docs/DataFrame.html). +## Features +- Type-safe column operations with compile-time guarantees +- Familiar, approachable API designed to feel easy coming from other languages. +- Interactive REPL for data exploration and plotting. + +## Quick start +Browse through some examples in [binder](https://mybinder.org/v2/gh/mchav/ihaskell-dataframe/HEAD) or in our [playground](https://ulwazi-exh9dbh2exbzgbc9.westus-01.azurewebsites.net/lab). + +## Install + +### Cabal +To use the CLI tool: +```bash +$ cabal update +$ cabal install dataframe +$ dataframe +``` + +As a prodject dependency add `dataframe` to your .cabal file. + +### Stack (in stack.yaml add to extra-deps if needed) +Add to your package.yaml dependencies: +```yaml +dependencies: + - dataframe +``` + +Or manually to stack.yaml extra-deps if needed. + +## Example + +```haskell +dataframe> df = D.fromNamedColumns [("product_id", D.fromList [1,1,2,2,3,3]), ("sales", D.fromList [100,120,50,20,40,30])] +dataframe> df +------------------ +product_id | sales +-----------|------ + Int | Int +-----------|------ +1 | 100 +1 | 120 +2 | 50 +2 | 20 +3 | 40 +3 | 30 + +dataframe> :exposeColumns df +"product_id :: Expr Int" +"sales :: Expr Int" +dataframe> df |> D.groupBy [F.name product_id] |> D.aggregate [F.sum sales `as` "total_sales"] +------------------------ +product_id | total_sales +-----------|------------ + Int | Int +-----------|------------ +1 | 220 +2 | 70 +3 | 70 +``` + +## Documentation +* 📚 User guide: https://dataframe.readthedocs.io/en/latest/ +* 📖 API reference: https://hackage.haskell.org/package/dataframe/docs/DataFrame.html diff --git a/app/Benchmark.hs b/app/Benchmark.hs --- a/app/Benchmark.hs +++ b/app/Benchmark.hs @@ -1,9 +1,11 @@ {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} import Data.Time import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D +import qualified DataFrame.Functions as F import System.Random.Stateful main :: IO () @@ -21,14 +23,14 @@ let generationTime = diffUTCTime endGeneration startGeneration putStrLn $ "Data generation Time: " ++ show generationTime startCalculation <- getCurrentTime - print $ D.mean "0" df - print $ D.variance "1" df + print $ D.mean (F.col @Double "0") df + print $ D.variance (F.col @Double "1") df print $ D.correlation "1" "2" df endCalculation <- getCurrentTime let calculationTime = diffUTCTime endCalculation startCalculation putStrLn $ "Calculation Time: " ++ show calculationTime startFilter <- getCurrentTime - print $ D.filter "0" (> (0.971 :: Double)) df D.|> D.take 10 + print $ D.filter (F.col @Double "0") (> 0.971) df D.|> D.take 10 endFilter <- getCurrentTime let filterTime = diffUTCTime endFilter startFilter putStrLn $ "Filter Time: " ++ show filterTime diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -32,6 +32,8 @@ "" putStrLn output + putStrLn =<< readProcess "cabal" ["update"] "" + let command = "cabal" args = [ "repl" diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 0.3.3.6 +version: 0.3.3.7 synopsis: A fast, safe, and intuitive DataFrame library. @@ -59,8 +59,8 @@ DataFrame.Operations.Core, DataFrame.Operations.Join, DataFrame.Operations.Merge, + DataFrame.Operations.Permutation, DataFrame.Operations.Subset, - DataFrame.Operations.Sorting, DataFrame.Operations.Statistics, DataFrame.Operations.Transformations, DataFrame.Operations.Typing, @@ -162,6 +162,7 @@ Operations.Filter, Operations.GroupBy, Operations.InsertColumn, + Operations.Join, Operations.Merge, Operations.ReadCsv, Operations.Sort, diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -22,9 +22,8 @@ macros for writing safe code. @ -\$ curl --output dataframe \"https:\/\/raw.githubusercontent.com\/mchav\/dataframe\/refs\/heads\/main\/scripts\/dataframe.sh\" -\$ chmod +x dataframe -\$ export PATH=$PATH:$PWD/dataframe +\$ cabal update +\$ cabal install dataframe \$ dataframe Configuring library for fake-package-0... Warning: No exposed modules @@ -115,6 +114,7 @@ __Row ops__ + * @D.filter :: Expr a -> (a -> Bool) -> DataFrame -> DataFrame@ * @D.filterWhere :: Expr Bool -> DataFrame -> DataFrame@ * @D.sortBy :: SortOrder -> [Text] -> DataFrame -> DataFrame@ @@ -128,11 +128,11 @@ __Group & aggregate__ * @D.groupBy :: [Text] -> DataFrame -> GroupedDataFrame@ - * @D.aggregate :: [(Text, F.UExpr)] -> GroupedDataFrame -> DataFrame@ + * @D.aggregate :: [NamedExpr] -> GroupedDataFrame -> DataFrame@ __Joins__ - * @D.innerJoin \/ D.leftJoin \/ D.rightJoin \/ D.fullJoin@ + * @D.innerJoin \/ D.leftJoin \/ D.rightJoin \/ D.fullOuterJoin@ == Expression DSL (F.*) at a glance Columns (typed): @@ -216,7 +216,7 @@ module Subset, module Transformations, module Aggregation, - module Sorting, + module Permutation, module Merge, module Join, module Statistics, @@ -305,7 +305,11 @@ ) import DataFrame.Operations.Join as Join import DataFrame.Operations.Merge as Merge -import DataFrame.Operations.Sorting as Sorting +import DataFrame.Operations.Permutation as Permutation ( + SortOrder (..), + shuffle, + sortBy, + ) import DataFrame.Operations.Statistics as Statistics ( correlation, frequencies, @@ -331,6 +335,7 @@ exclude, filter, filterAllJust, + filterAllNothing, filterBy, filterJust, filterNothing, diff --git a/src/DataFrame/Display/Terminal/Plot.hs b/src/DataFrame/Display/Terminal/Plot.hs --- a/src/DataFrame/Display/Terminal/Plot.hs +++ b/src/DataFrame/Display/Terminal/Plot.hs @@ -24,6 +24,7 @@ import DataFrame.Internal.Column (Column (..), isNumeric) import qualified DataFrame.Internal.Column as D import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) +import DataFrame.Internal.Expression import DataFrame.Operations.Core import qualified DataFrame.Operations.Subset as D import Granite @@ -52,14 +53,14 @@ } plotHistogram :: (HasCallStack) => T.Text -> DataFrame -> IO () -plotHistogram colName = plotHistogramWith colName (defaultPlotConfig Histogram) +plotHistogram colName = plotHistogramWith colName 30 (defaultPlotConfig Histogram) plotHistogramWith :: - (HasCallStack) => T.Text -> PlotConfig -> DataFrame -> IO () -plotHistogramWith colName config df = do + (HasCallStack) => T.Text -> Int -> PlotConfig -> DataFrame -> IO () +plotHistogramWith colName numBins config df = do let values = extractNumericColumn colName df (minVal, maxVal) = if null values then (0, 1) else (minimum values, maximum values) - T.putStrLn $ histogram (bins 30 minVal maxVal) values (plotSettings config) + T.putStrLn $ histogram (bins numBins minVal maxVal) values (plotSettings config) plotScatter :: (HasCallStack) => T.Text -> T.Text -> DataFrame -> IO () plotScatter xCol yCol = plotScatterWith xCol yCol (defaultPlotConfig Scatter) @@ -82,7 +83,7 @@ let vals = extractStringColumn grouping df let df' = insertColumn grouping (D.fromList vals) df xs <- forM (L.nub vals) $ \col -> do - let filtered = D.filter grouping (== col) df' + let filtered = D.filter (Col grouping) (== col) df' xVals = extractNumericColumn xCol filtered yVals = extractNumericColumn yCol filtered points = zip xVals yVals diff --git a/src/DataFrame/Display/Web/Plot.hs b/src/DataFrame/Display/Web/Plot.hs --- a/src/DataFrame/Display/Web/Plot.hs +++ b/src/DataFrame/Display/Web/Plot.hs @@ -26,6 +26,7 @@ import DataFrame.Internal.Column (Column (..), isNumeric) import qualified DataFrame.Internal.Column as D import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) +import DataFrame.Internal.Expression import DataFrame.Operations.Core import qualified DataFrame.Operations.Subset as D import System.Directory @@ -39,6 +40,7 @@ std_out, waitForProcess, ) +import Text.Printf newtype HtmlPlot = HtmlPlot T.Text deriving (Show) @@ -94,18 +96,20 @@ , "px;height:" , T.pack (show height) , "px\">\n" - , "\n" + , "\n" , "\n" ] plotHistogram :: (HasCallStack) => T.Text -> DataFrame -> IO HtmlPlot -plotHistogram colName = plotHistogramWith colName (defaultPlotConfig Histogram) +plotHistogram colName = plotHistogramWith colName 30 (defaultPlotConfig Histogram) plotHistogramWith :: - (HasCallStack) => T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot -plotHistogramWith colName config df = do + (HasCallStack) => T.Text -> Int -> PlotConfig -> DataFrame -> IO HtmlPlot +plotHistogramWith colName numBins config df = do chartId <- generateChartId let values = extractNumericColumn colName df (minVal, maxVal) = if null values then (0, 1) else (minimum values, maximum values) @@ -115,7 +119,7 @@ counts = calculateHistogram values bins binWidth labels = - T.intercalate "," ["\"" <> T.pack (show (round b :: Int)) <> "\"" | b <- bins] + T.intercalate "," ["\"" <> T.pack (printf "%.2f" b) <> "\"" | b <- bins] dataPoints = T.intercalate "," [T.pack (show c) | c <- counts] chartTitle = @@ -125,9 +129,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -153,7 +159,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ @@ -186,9 +192,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"scatter\",\n" , " data: {\n" , " datasets: [{\n" @@ -215,7 +223,7 @@ , "\" } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ @@ -246,7 +254,7 @@ ] datasets <- forM (zip uniqueVals colors) $ \(val, color) -> do - let filtered = D.filter grouping (== val) df' + let filtered = D.filter (Col grouping) (== val) df' xVals = extractNumericColumn xCol filtered yVals = extractNumericColumn yCol filtered points = zip xVals yVals @@ -279,9 +287,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"scatter\",\n" , " data: {\n" , " datasets: [\n" @@ -301,7 +311,7 @@ , "\" } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ @@ -353,9 +363,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"line\",\n" , " data: {\n" , " labels: [" @@ -375,7 +387,7 @@ , "\" } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ @@ -407,9 +419,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -433,7 +447,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ HtmlPlot $ @@ -451,8 +465,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , "\", {\n" , " type: \"bar\",\n" , " data: {\n" @@ -477,7 +494,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ HtmlPlot $ @@ -502,9 +519,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"pie\",\n" , " data: {\n" , " labels: [" @@ -524,7 +543,7 @@ , chartTitle , "\" }\n" , " }\n" - , "});" + , "});});" ] return $ HtmlPlot $ @@ -546,9 +565,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"pie\",\n" , " data: {\n" , " labels: [" @@ -568,7 +589,7 @@ , chartTitle , "\" }\n" , " }\n" - , "});" + , "});});" ] return $ HtmlPlot $ @@ -637,9 +658,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -658,7 +681,7 @@ , " yAxes: [{ stacked: true, ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ @@ -689,9 +712,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -715,7 +740,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ @@ -748,9 +773,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -776,7 +803,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ HtmlPlot $ @@ -800,9 +827,11 @@ jsCode = T.concat - [ "new Chart(\"" + [ "requirejs(['https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js'], function (Chart) {\n" + , "var ctx = document.getElementById('" , chartId - , "\", {\n" + , "').getContext('2d');\n" + , "new Chart(ctx , {\n" , " type: \"bar\",\n" , " data: {\n" , " labels: [" @@ -826,7 +855,7 @@ , " yAxes: [{ ticks: { beginAtZero: true } }]\n" , " }\n" , " }\n" - , "});" + , "});});" ] return $ HtmlPlot $ diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs --- a/src/DataFrame/Functions.hs +++ b/src/DataFrame/Functions.hs @@ -22,6 +22,7 @@ ) import DataFrame.Internal.Expression ( Expr (..), + NamedExpr, UExpr (..), eSize, interpret, @@ -38,7 +39,7 @@ import Data.Function import qualified Data.List as L import qualified Data.Map as M -import Data.Maybe (fromMaybe, listToMaybe) +import Data.Maybe (listToMaybe) import qualified Data.Set as S import qualified Data.Text as T import Data.Type.Equality @@ -62,9 +63,13 @@ col :: (Columnable a) => T.Text -> Expr a col = Col -as :: (Columnable a) => Expr a -> T.Text -> (T.Text, UExpr) +as :: (Columnable a) => Expr a -> T.Text -> NamedExpr as expr name = (name, Wrap expr) +infixr 0 .= +(.=) :: (Columnable a) => T.Text -> Expr a -> NamedExpr +(.=) = flip as + ifThenElse :: (Columnable a) => Expr Bool -> Expr a -> Expr a -> Expr a ifThenElse = If @@ -200,6 +205,22 @@ (Columnable a, Columnable b) => Expr b -> a -> (a -> b -> a) -> Expr a reduce expr = AggFold expr "foldUdf" +whenPresent :: + forall a b. + (Columnable a, Columnable b) => (a -> b) -> Expr (Maybe a) -> Expr (Maybe b) +whenPresent f = lift (fmap f) + +whenBothPresent :: + forall a b c. + (Columnable a, Columnable b, Columnable c) => + (a -> b -> c) -> Expr (Maybe a) -> Expr (Maybe b) -> Expr (Maybe c) +whenBothPresent f = lift2 (\l r -> f <$> l <*> r) + +recode :: + forall a b. + (Columnable a, Columnable b) => [(a, b)] -> Expr a -> Expr (Maybe b) +recode mapping = UnaryOp (T.pack ("recode " ++ show mapping)) (`lookup` mapping) + generateConditions :: TypedColumn Double -> [Expr Bool] -> [Expr Double] -> DataFrame -> [Expr Bool] generateConditions labels conds ps df = @@ -222,75 +243,14 @@ pickTopNBool df labels (deduplicate df expandedConds) generatePrograms :: - [Expr Bool] -> [Expr Double] -> [Expr Double] -> [Expr Double] -> [Expr Double] -generatePrograms conds vars' constants [] = - let - vars = vars' ++ constants - in - nubOrd $ - vars - ++ [ transform p - | p <- vars' - , transform <- - [ abs - , sqrt - , log . (+ Lit 1) - , exp - , sin - , cos - , relu - , signum - ] - ] - ++ [ pow i p - | p <- vars - , i <- [2 .. 6] - ] - ++ [ p + q - | (i, p) <- zip [0 ..] vars - , (j, q) <- zip [0 ..] vars - , Prelude.not (isLiteral p && isLiteral q) - , i > j - ] - ++ [ DataFrame.Functions.min p q - | (i, p) <- zip [0 ..] vars - , (j, q) <- zip [0 ..] vars - , p /= q - , Prelude.not (isLiteral p && isLiteral q) - , i > j - ] - ++ [ DataFrame.Functions.max p q - | (i, p) <- zip [0 ..] vars - , (j, q) <- zip [0 ..] vars - , p /= q - , Prelude.not (isLiteral p && isLiteral q) - , i > j - ] - ++ [ ifThenElse cond r s - | cond <- conds - , r <- vars - , s <- vars - , r /= s - ] - ++ [ p - q - | (i, p) <- zip [0 ..] vars - , (j, q) <- zip [0 ..] vars - , Prelude.not (isLiteral p && isLiteral q) - , i /= j - ] - ++ [ p * q - | (i, p) <- zip [0 ..] vars - , (j, q) <- zip [0 ..] vars - , Prelude.not (isLiteral p && isLiteral q) - , i >= j - ] - ++ [ p / q - | (i, p) <- zip [0 ..] vars - , (j, q) <- zip [0 ..] vars - , Prelude.not (isLiteral p && isLiteral q) - , i /= j - ] -generatePrograms conds vars constants ps = + Bool -> + [Expr Bool] -> + [Expr Double] -> + [Expr Double] -> + [Expr Double] -> + [Expr Double] +generatePrograms _ _ vars' constants [] = vars' ++ constants +generatePrograms includeConds conds vars constants ps = let existingPrograms = ps ++ vars ++ constants in @@ -318,26 +278,30 @@ , Prelude.not (isLiteral p && isLiteral q) , i >= j ] - ++ [ DataFrame.Functions.min p q - | (i, p) <- zip [0 ..] existingPrograms - , (j, q) <- zip [0 ..] existingPrograms - , Prelude.not (isLiteral p && isLiteral q) - , p /= q - , i > j - ] - ++ [ DataFrame.Functions.max p q - | (i, p) <- zip [0 ..] existingPrograms - , (j, q) <- zip [0 ..] existingPrograms - , Prelude.not (isLiteral p && isLiteral q) - , p /= q - , i > j - ] - ++ [ ifThenElse cond r s - | cond <- conds - , r <- existingPrograms - , s <- existingPrograms - , r /= s - ] + ++ ( if includeConds + then + [ DataFrame.Functions.min p q + | (i, p) <- zip [0 ..] existingPrograms + , (j, q) <- zip [0 ..] existingPrograms + , Prelude.not (isLiteral p && isLiteral q) + , p /= q + , i > j + ] + ++ [ DataFrame.Functions.max p q + | (i, p) <- zip [0 ..] existingPrograms + , (j, q) <- zip [0 ..] existingPrograms + , Prelude.not (isLiteral p && isLiteral q) + , p /= q + , i > j + ] + ++ [ ifThenElse cond r s + | cond <- conds + , r <- existingPrograms + , s <- existingPrograms + , r /= s + ] + else [] + ) ++ [ p - q | (i, p) <- zip [0 ..] existingPrograms , (j, q) <- zip [0 ..] existingPrograms @@ -456,7 +420,7 @@ in case beamSearch df' - (BeamConfig d b F1) + (BeamConfig d b F1 True) t (percentiles df' ++ [lit 1, lit 0, lit (-1)]) [] @@ -469,10 +433,23 @@ let doubleColumns = map (either throw id . (`columnAsDoubleVector` df)) (D.columnNames df) in - concatMap (\c -> map (lit . (`percentile'` c)) [1, 23, 75, 99]) doubleColumns - ++ map (lit . variance') doubleColumns - ++ map (lit . sqrt . variance') doubleColumns + concatMap + (\c -> map (lit . roundTo2SigDigits . (`percentile'` c)) [1, 25, 75, 99]) + doubleColumns + ++ map (lit . roundTo2SigDigits . variance') doubleColumns + ++ map (lit . roundTo2SigDigits . sqrt . variance') doubleColumns +roundToSigDigits :: Int -> Double -> Double +roundToSigDigits n x + | x == 0 = 0 + | otherwise = + let magnitude = floor (logBase 10 (abs x)) + scale = 10 ** fromIntegral (n - 1 - magnitude) + in fromIntegral (round (x * scale)) / scale + +roundTo2SigDigits :: Double -> Double +roundTo2SigDigits = roundToSigDigits 2 + fitRegression :: -- | Target expression T.Text -> @@ -485,7 +462,7 @@ fitRegression target d b df = let df' = exclude [target] df - targetMean = fromMaybe 0 $ Stats.mean target df + targetMean = Stats.mean (Col @Double target) df t = case interpret df (Col target) of Left e -> throw e Right v -> v @@ -496,6 +473,7 @@ d b MutualInformation + False ) t (percentiles df') @@ -509,7 +487,7 @@ ( D.derive "_generated_regression_feature_" p df & select ["_generated_regression_feature_"] ) - (BeamConfig d b MeanSquaredError) + (BeamConfig d b MeanSquaredError False) t (percentiles df' ++ [lit targetMean, lit 10]) [] @@ -541,10 +519,11 @@ { searchDepth :: Int , beamLength :: Int , lossFunction :: LossFunction + , includeConditionals :: Bool } defaultBeamConfig :: BeamConfig -defaultBeamConfig = BeamConfig 2 100 PearsonCorrelation +defaultBeamConfig = BeamConfig 2 100 PearsonCorrelation False beamSearch :: DataFrame -> @@ -570,7 +549,7 @@ outputs constants conditions - (generatePrograms conditions vars constants ps) + (generatePrograms (includeConditionals cfg) conditions vars constants ps) where vars = map col names conditions = generateConditions outputs conds (vars ++ constants ++ ps) df @@ -648,7 +627,7 @@ Right v -> v ordered = Prelude.take - 100 + 10 ( map fst $ L.sortBy ( \(_, c2) (_, c1) -> @@ -766,7 +745,7 @@ lhs <- typeFromString [t1] rhs <- typeFromString [t2] return (AppT (AppT outer lhs) rhs) -typeFromString s = fail $ "Unsupported type: " ++ unwords s +typeFromString s = fail $ "Unsupported types: " ++ unwords s declareColumns :: DataFrame -> DecsQ declareColumns df = diff --git a/src/DataFrame/IO/CSV.hs b/src/DataFrame/IO/CSV.hs --- a/src/DataFrame/IO/CSV.hs +++ b/src/DataFrame/IO/CSV.hs @@ -228,7 +228,7 @@ let firstRow = parseLine sep firstLine columnNames = case headerSpec opts of NoHeader -> map (T.pack . show) [0 .. length firstRow - 1] - UseFirstRow -> map (T.filter (/= '\"') . TE.decodeUtf8Lenient) firstRow + UseFirstRow -> map (stripQuotes . TE.decodeUtf8Lenient) firstRow ProvideNames ns -> ns unless (headerSpec opts == UseFirstRow) $ hSeek handle AbsoluteSeek 0 @@ -291,7 +291,7 @@ where processValue :: Int -> BS.ByteString -> GrowingColumn -> IO () processValue !idx !bs !col = do - let !val = TE.decodeUtf8Lenient bs + let !val = (stripQuotes . TE.decodeUtf8Lenient) bs case col of GrowingInt gv nulls -> case readByteStringInt bs of @@ -478,3 +478,12 @@ ++ " has less items than " ++ "the other columns at index " ++ show i + +stripQuotes :: T.Text -> T.Text +stripQuotes txt = + case T.uncons txt of + Just ('"', rest) -> + case T.unsnoc rest of + Just (middle, '"') -> middle + _ -> txt + _ -> txt diff --git a/src/DataFrame/IO/Unstable/CSV.hs b/src/DataFrame/IO/Unstable/CSV.hs --- a/src/DataFrame/IO/Unstable/CSV.hs +++ b/src/DataFrame/IO/Unstable/CSV.hs @@ -46,6 +46,7 @@ ReadOptions (..), defaultReadOptions, shouldInferFromSample, + stripQuotes, typeInferenceSampleSize, ) import DataFrame.Internal.DataFrame (DataFrame (..)) @@ -94,7 +95,7 @@ ) UseFirstRow -> ( Vector.fromList $ - map extractField' [0 .. numCol - 1] + map (stripQuotes . extractField') [0 .. numCol - 1] , 1 ) ProvideNames ns -> @@ -107,8 +108,8 @@ else 0 in parseFromExamples n - (dateFormat opts) (safeRead opts) + (dateFormat opts) col generateColumn col = parseTypes $ @@ -116,7 +117,7 @@ numRow ( map ( \row -> - extractField' + (stripQuotes . extractField') (row * numCol + col) ) [dataStartRow .. totalRows - 1] diff --git a/src/DataFrame/Internal/Column.hs b/src/DataFrame/Internal/Column.hs --- a/src/DataFrame/Internal/Column.hs +++ b/src/DataFrame/Internal/Column.hs @@ -334,6 +334,20 @@ atIndicesStable indexes (OptionalColumn column) = OptionalColumn $ VG.unsafeBackpermute column (VG.convert indexes) {-# INLINE atIndicesStable #-} +atIndicesWithNulls :: VB.Vector (Maybe Int) -> Column -> Column +atIndicesWithNulls indices column + | VB.all isJust indices = + atIndicesStable + (VU.fromList (catMaybes (VB.toList indices))) + column + | otherwise = case column of + BoxedColumn col -> + OptionalColumn $ VB.map (fmap (col VB.!)) indices + UnboxedColumn col -> + OptionalColumn $ VB.map (fmap (col VU.!)) indices + OptionalColumn col -> + OptionalColumn $ VB.map (\ix -> ix >>= (col VB.!)) indices + -- | Internal helper to get indices in a boxed vector. getIndices :: VU.Vector Int -> VB.Vector a -> VB.Vector a getIndices indices xs = VB.generate (VU.length indices) (\i -> xs VB.! (indices VU.! i)) @@ -1229,6 +1243,34 @@ { userType = Right (typeRep @Int) , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) , callingFunctionName = Just "toIntVector" + , errorColumnName = Nothing + } + ) + +toUnboxedVector :: + forall a. + (Columnable a, VU.Unbox a) => Column -> Either DataFrameException (VU.Vector a) +toUnboxedVector column = + case column of + UnboxedColumn (f :: VU.Vector b) -> case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right f + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Int) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "toIntVector" + , errorColumnName = Nothing + } + ) + _ -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) + , callingFunctionName = Just "toUnboxedVector" , errorColumnName = Nothing } ) diff --git a/src/DataFrame/Internal/DataFrame.hs b/src/DataFrame/Internal/DataFrame.hs --- a/src/DataFrame/Internal/DataFrame.hs +++ b/src/DataFrame/Internal/DataFrame.hs @@ -297,3 +297,9 @@ columnAsFloatVector :: T.Text -> DataFrame -> Either DataFrameException (VU.Vector Float) columnAsFloatVector name df = toFloatVector (unsafeGetColumn name df) + +columnAsUnboxedVector :: + forall a. + (Columnable a, VU.Unbox a) => + T.Text -> DataFrame -> Either DataFrameException (VU.Vector a) +columnAsUnboxedVector name df = toUnboxedVector @a (unsafeGetColumn name df) diff --git a/src/DataFrame/Internal/Expression.hs b/src/DataFrame/Internal/Expression.hs --- a/src/DataFrame/Internal/Expression.hs +++ b/src/DataFrame/Internal/Expression.hs @@ -95,6 +95,8 @@ data UExpr where Wrap :: (Columnable a) => Expr a -> UExpr +type NamedExpr = (T.Text, UExpr) + interpret :: forall a. (Columnable a) => diff --git a/src/DataFrame/Internal/Parsing.hs b/src/DataFrame/Internal/Parsing.hs --- a/src/DataFrame/Internal/Parsing.hs +++ b/src/DataFrame/Internal/Parsing.hs @@ -13,9 +13,11 @@ import Text.Read (readMaybe) isNullish :: T.Text -> Bool -isNullish s = - s - `S.member` S.fromList ["Nothing", "NULL", "", " ", "nan", "null", "N/A", "NaN", "NAN"] +isNullish = + ( `S.member` + S.fromList + ["Nothing", "NULL", "", " ", "nan", "null", "N/A", "NaN", "NAN", "NA"] + ) readValue :: (HasCallStack, Read a) => T.Text -> a readValue s = case readMaybe (T.unpack s) of diff --git a/src/DataFrame/Internal/Statistics.hs b/src/DataFrame/Internal/Statistics.hs --- a/src/DataFrame/Internal/Statistics.hs +++ b/src/DataFrame/Internal/Statistics.hs @@ -58,9 +58,10 @@ -- accumulator: count, mean, m2, m3 data SkewAcc = SkewAcc !Int !Double !Double !Double deriving (Show) -skewnessStep :: SkewAcc -> Double -> SkewAcc -skewnessStep (SkewAcc !n !mean !m2 !m3) !x = +skewnessStep :: (VU.Unbox a, Num a, Real a) => SkewAcc -> a -> SkewAcc +skewnessStep (SkewAcc !n !mean !m2 !m3) !x' = let !n' = n + 1 + x = realToFrac x' !k = fromIntegral n' !delta = x - mean !mean' = mean + delta / k @@ -75,7 +76,7 @@ | otherwise = (sqrt (fromIntegral n - 1) * m3) / sqrt (m2 ^ 3) {-# INLINE computeSkewness #-} -skewness' :: VU.Vector Double -> Double +skewness' :: (VU.Unbox a, Real a, Num a) => VU.Vector a -> Double skewness' = computeSkewness . VU.foldl' skewnessStep (SkewAcc 0 0 0 0) {-# INLINE skewness' #-} @@ -106,7 +107,9 @@ | otherwise = (sumX, sumY, sumSquaredX, sumSquaredY, sumXY) {-# INLINE correlation' #-} -quantiles' :: VU.Vector Int -> Int -> VU.Vector Double -> VU.Vector Double +quantiles' :: + (VU.Unbox a, Num a, Real a) => + VU.Vector Int -> Int -> VU.Vector a -> VU.Vector Double quantiles' qs q samp | VU.null samp = throw $ EmptyDataSetException "quantiles" | q < 2 = throw $ WrongQuantileNumberException q @@ -121,20 +124,20 @@ !position = p * fromIntegral (n - 1) !index = floor position !f = position - fromIntegral index - x <- VUM.read mutableSamp index + x <- fmap realToFrac (VUM.read mutableSamp index) if f == 0 then return x else do - y <- VUM.read mutableSamp (index + 1) + y <- fmap realToFrac (VUM.read mutableSamp (index + 1)) return $ (1 - f) * x + f * y ) qs {-# INLINE quantiles' #-} -percentile' :: Int -> VU.Vector Double -> Double +percentile' :: (VU.Unbox a, Num a, Real a) => Int -> VU.Vector a -> Double percentile' n = VU.head . quantiles' (VU.fromList [n]) 100 -interQuartileRange' :: VU.Vector Double -> Double +interQuartileRange' :: (VU.Unbox a, Num a, Real a) => VU.Vector a -> Double interQuartileRange' samp = let quartiles = quantiles' (VU.fromList [1, 3]) 4 samp in quartiles VU.! 1 - quartiles VU.! 0 diff --git a/src/DataFrame/Operations/Aggregation.hs b/src/DataFrame/Operations/Aggregation.hs --- a/src/DataFrame/Operations/Aggregation.hs +++ b/src/DataFrame/Operations/Aggregation.hs @@ -113,7 +113,7 @@ {- | Aggregate a grouped dataframe using the expressions given. All ungrouped columns will be dropped. -} -aggregate :: [(T.Text, UExpr)] -> GroupedDataFrame -> DataFrame +aggregate :: [NamedExpr] -> GroupedDataFrame -> DataFrame aggregate aggs gdf@(Grouped df groupingColumns valueIndices offsets) = let df' = diff --git a/src/DataFrame/Operations/Core.hs b/src/DataFrame/Operations/Core.hs --- a/src/DataFrame/Operations/Core.hs +++ b/src/DataFrame/Operations/Core.hs @@ -32,6 +32,7 @@ fromVector, ) import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn) +import DataFrame.Internal.Expression import DataFrame.Internal.Parsing (isNullish) import DataFrame.Internal.Row (Any, mkColumnFromRow) import Type.Reflection @@ -521,8 +522,8 @@ @ -} -valueCounts :: forall a. (Columnable a) => T.Text -> DataFrame -> [(a, Int)] -valueCounts columnName df = case getColumn columnName df of +valueCounts :: forall a. (Columnable a) => Expr a -> DataFrame -> [(a, Int)] +valueCounts (Col columnName) df = case getColumn columnName df of Nothing -> throw $ ColumnNotFoundException columnName "valueCounts" (M.keys $ columnIndices df) @@ -575,6 +576,7 @@ } ) Just Refl -> M.toAscList column +valueCounts _ _ = error "Cannot call value counts on non-column reference" {- | A left fold for dataframes that takes the dataframe as the last object. This makes it easier to chain operations. diff --git a/src/DataFrame/Operations/Join.hs b/src/DataFrame/Operations/Join.hs --- a/src/DataFrame/Operations/Join.hs +++ b/src/DataFrame/Operations/Join.hs @@ -1,4 +1,6 @@ +{-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} module DataFrame.Operations.Join where @@ -29,14 +31,32 @@ DataFrame -> -- Left hand side DataFrame join INNER xs right = innerJoin xs right -join LEFT xs right = error "UNIMPLEMENTED" -join RIGHT xs right = error "UNIMPLEMENTED" -join FULL_OUTER xs right = error "UNIMPLEMENTED" +join LEFT xs right = leftJoin xs right +join RIGHT xs right = rightJoin xs right +join FULL_OUTER xs right = fullOuterJoin xs right -{- | Inner join of two dataframes. Note: for chaining, the left dataframe is actually -on the right side. +{- | Performs an inner join on two dataframes using the specified key columns. +Returns only rows where the key values exist in both dataframes. + +==== __Example__ +@ +ghci> df = D.fromNamedColumns [("key", D.fromList ["K0", "K1", "K2", "K3"]), ("A", D.fromList ["A0", "A1", "A2", "A3"])] +ghci> other = D.fromNamedColumns [("key", D.fromList ["K0", "K1", "K2"]), ("B", D.fromList ["B0", "B1", "B2"])] +ghci> D.innerJoin ["key"] df other + +----------------- + key | A | B +------|-----|---- + Text | Text| Text +------|-----|---- + K0 | A0 | B0 + K1 | A1 | B1 + K2 | A2 | B2 + +@ -} -innerJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame +innerJoin :: + [T.Text] -> DataFrame -> DataFrame -> DataFrame innerJoin cs right left = let leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) @@ -90,6 +110,233 @@ (VU.length expandedRightIndicies, snd (D.dataframeDimensions right)) } -- [string] + leftColumns = D.columnNames left + rightColumns = D.columnNames right + initDf = expandedLeft + insertIfPresent _ Nothing df = df + insertIfPresent name (Just c) df = D.insertColumn name c df + in + D.fold + ( \name df -> + if name `elem` cs + then df + else + ( if name `elem` leftColumns + then insertIfPresent ("Right_" <> name) (D.getColumn name expandedRight) df + else insertIfPresent name (D.getColumn name expandedRight) df + ) + ) + rightColumns + initDf + +{- | Performs a left join on two dataframes using the specified key columns. +Returns all rows from the left dataframe, with matching rows from the right dataframe. +Non-matching rows will have Nothing/null values for columns from the right dataframe. + +==== __Example__ +@ +ghci> df = D.fromNamedColumns [("key", D.fromList ["K0", "K1", "K2", "K3"]), ("A", D.fromList ["A0", "A1", "A2", "A3"])] +ghci> other = D.fromNamedColumns [("key", D.fromList ["K0", "K1", "K2"]), ("B", D.fromList ["B0", "B1", "B2"])] +ghci> D.leftJoin ["key"] df other + +------------------------ + key | A | B +------|-----|---------- + Text | Text| Maybe Text +------|-----|---------- + K0 | A0 | Just "B0" + K1 | A1 | Just "B1" + K2 | A2 | Just "B2" + K3 | A3 | Nothing + +@ +-} +leftJoin :: + [T.Text] -> DataFrame -> DataFrame -> DataFrame +leftJoin cs right left = + let + leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) + leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) + rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right) + rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) + rightKeyCountsAndIndices = + VU.foldr + (\(i, v) acc -> M.insertWith (++) v [i] acc) + M.empty + (VU.indexed rightRowRepresentations) + rightKeyCountsAndIndicesVec = M.map VU.fromList rightKeyCountsAndIndices + leftRowCount = fst (D.dimensions left) + pairs = + [ (i, maybeRight) + | i <- [0 .. leftRowCount - 1] + , maybeRight <- + case M.lookup (leftRowRepresentations VU.! i) rightKeyCountsAndIndicesVec of + Nothing -> [Nothing] + Just rVec -> map Just (VU.toList rVec) + ] + expandedLeftIndicies = VU.fromList (map fst pairs) + expandedRightIndicies = VB.fromList (map snd pairs) + expandedLeft = + left + { columns = VB.map (D.atIndicesStable expandedLeftIndicies) (D.columns left) + , dataframeDimensions = + (VU.length expandedLeftIndicies, snd (D.dataframeDimensions left)) + } + expandedRight = + right + { columns = VB.map (D.atIndicesWithNulls expandedRightIndicies) (D.columns right) + , dataframeDimensions = + (VB.length expandedRightIndicies, snd (D.dataframeDimensions right)) + } + leftColumns = D.columnNames left + rightColumns = D.columnNames right + initDf = expandedLeft + insertIfPresent _ Nothing df = df + insertIfPresent name (Just c) df = D.insertColumn name c df + in + D.fold + ( \name df -> + if name `elem` cs + then df + else + ( if name `elem` leftColumns + then insertIfPresent ("Right_" <> name) (D.getColumn name expandedRight) df + else insertIfPresent name (D.getColumn name expandedRight) df + ) + ) + rightColumns + initDf + +{- | Performs a right join on two dataframes using the specified key columns. +Returns all rows from the right dataframe, with matching rows from the left dataframe. +Non-matching rows will have Nothing/null values for columns from the left dataframe. + +==== __Example__ +@ +ghci> df = D.fromNamedColumns [("key", D.fromList ["K0", "K1", "K2", "K3"]), ("A", D.fromList ["A0", "A1", "A2", "A3"])] +ghci> other = D.fromNamedColumns [("key", D.fromList ["K0", "K1"]), ("B", D.fromList ["B0", "B1"])] +ghci> D.rightJoin ["key"] df other + +----------------- + key | A | B +------|-----|---- + Text | Text| Text +------|-----|---- + K0 | A0 | B0 + K1 | A1 | B1 + +@ +-} +rightJoin :: + [T.Text] -> DataFrame -> DataFrame -> DataFrame +rightJoin cs right left = + let + leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) + leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) + leftKeyCountsAndIndicesVec = + M.map VU.fromList $ + VU.foldr + (\(i, v) acc -> M.insertWith (++) v [i] acc) + M.empty + (VU.indexed leftRowRepresentations) + rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right) + rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) + rightRowCount = fst (D.dimensions right) + pairs = + [ (maybeLeft, j) + | j <- [0 .. rightRowCount - 1] + , maybeLeft <- + case M.lookup (rightRowRepresentations VU.! j) leftKeyCountsAndIndicesVec of + Nothing -> [Nothing] + Just lVec -> map Just (VU.toList lVec) + ] + expandedLeftIndicies = VB.fromList (map fst pairs) + expandedRightIndicies = VU.fromList (map snd pairs) + expandedLeft = + left + { columns = VB.map (D.atIndicesWithNulls expandedLeftIndicies) (D.columns left) + , dataframeDimensions = + (VB.length expandedLeftIndicies, snd (D.dataframeDimensions left)) + } + expandedRight = + right + { columns = VB.map (D.atIndicesStable expandedRightIndicies) (D.columns right) + , dataframeDimensions = + (VU.length expandedRightIndicies, snd (D.dataframeDimensions right)) + } + leftColumns = D.columnNames left + rightColumns = D.columnNames right + initDf = expandedLeft + insertIfPresent _ Nothing df = df + insertIfPresent name (Just c) df = D.insertColumn name c df + in + D.fold + ( \name df -> + if name `elem` cs + then df + else + ( if name `elem` leftColumns + then insertIfPresent ("Right_" <> name) (D.getColumn name expandedRight) df + else insertIfPresent name (D.getColumn name expandedRight) df + ) + ) + rightColumns + initDf + +fullOuterJoin :: + [T.Text] -> DataFrame -> DataFrame -> DataFrame +fullOuterJoin cs right left = + let + leftIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices left) + leftRowRepresentations = VU.generate (fst (D.dimensions left)) (D.mkRowRep leftIndicesToGroup left) + leftKeyCountsAndIndices = + VU.foldr + (\(i, v) acc -> M.insertWith (++) v [i] acc) + M.empty + (VU.indexed leftRowRepresentations) + leftKeyCountsAndIndicesVec = M.map VU.fromList leftKeyCountsAndIndices + rightIndicesToGroup = M.elems $ M.filterWithKey (\k _ -> k `elem` cs) (D.columnIndices right) + rightRowRepresentations = VU.generate (fst (D.dimensions right)) (D.mkRowRep rightIndicesToGroup right) + rightKeyCountsAndIndices = + VU.foldr + (\(i, v) acc -> M.insertWith (++) v [i] acc) + M.empty + (VU.indexed rightRowRepresentations) + rightKeyCountsAndIndicesVec = M.map VU.fromList rightKeyCountsAndIndices + matchedPairs = + concatMap + ( \(lVec, rVec) -> + [ (Just lIdx, Just rIdx) + | lIdx <- VU.toList lVec + , rIdx <- VU.toList rVec + ] + ) + ( M.elems + (M.intersectionWith (,) leftKeyCountsAndIndicesVec rightKeyCountsAndIndicesVec) + ) + leftOnlyPairs = + concatMap + (map (\lIdx -> (Just lIdx, Nothing)) . VU.toList) + (M.elems (leftKeyCountsAndIndicesVec `M.difference` rightKeyCountsAndIndicesVec)) + rightOnlyPairs = + concatMap + (map (\rIdx -> (Nothing, Just rIdx)) . VU.toList) + (M.elems (rightKeyCountsAndIndicesVec `M.difference` leftKeyCountsAndIndicesVec)) + pairs = matchedPairs ++ leftOnlyPairs ++ rightOnlyPairs + expandedLeftIndicies = VB.fromList (map fst pairs) + expandedRightIndicies = VB.fromList (map snd pairs) + expandedLeft = + left + { columns = VB.map (D.atIndicesWithNulls expandedLeftIndicies) (D.columns left) + , dataframeDimensions = + (VB.length expandedLeftIndicies, snd (D.dataframeDimensions left)) + } + expandedRight = + right + { columns = VB.map (D.atIndicesWithNulls expandedRightIndicies) (D.columns right) + , dataframeDimensions = + (VB.length expandedRightIndicies, snd (D.dataframeDimensions right)) + } leftColumns = D.columnNames left rightColumns = D.columnNames right initDf = expandedLeft diff --git a/src/DataFrame/Operations/Permutation.hs b/src/DataFrame/Operations/Permutation.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Operations/Permutation.hs @@ -0,0 +1,55 @@ +{-# LANGUAGE OverloadedStrings #-} + +module DataFrame.Operations.Permutation where + +import qualified Data.List as L +import qualified Data.Text as T +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU + +import Control.Exception (throw) +import DataFrame.Errors (DataFrameException (..)) +import DataFrame.Internal.Column +import DataFrame.Internal.DataFrame (DataFrame (..)) +import DataFrame.Internal.Row +import DataFrame.Operations.Core +import System.Random + +-- | Sort order taken as a parameter by the 'sortBy' function. +data SortOrder = Ascending | Descending deriving (Eq) + +{- | O(k log n) Sorts the dataframe by a given row. + +> sortBy Ascending ["Age"] df +-} +sortBy :: + SortOrder -> + [T.Text] -> + DataFrame -> + DataFrame +sortBy order names df + | any (`notElem` columnNames df) names = + throw $ + ColumnNotFoundException + (T.pack $ show $ names L.\\ columnNames df) + "sortBy" + (columnNames df) + | otherwise = + let + indexes = sortedIndexes' (order == Ascending) (toRowVector names df) + in + df{columns = V.map (atIndicesStable indexes) (columns df)} + +shuffle :: + (RandomGen g) => + g -> + DataFrame -> + DataFrame +shuffle pureGen df = + let + indexes = shuffledIndices pureGen (fst (dimensions df)) + in + df{columns = V.map (atIndicesStable indexes) (columns df)} + +shuffledIndices :: (RandomGen g) => g -> Int -> VU.Vector Int +shuffledIndices pureGen k = VU.fromList (fst (uniformShuffleList [0 .. (k - 1)] pureGen)) diff --git a/src/DataFrame/Operations/Sorting.hs b/src/DataFrame/Operations/Sorting.hs deleted file mode 100644 --- a/src/DataFrame/Operations/Sorting.hs +++ /dev/null @@ -1,39 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} - -module DataFrame.Operations.Sorting where - -import qualified Data.List as L -import qualified Data.Text as T -import qualified Data.Vector as V - -import Control.Exception (throw) -import DataFrame.Errors (DataFrameException (..)) -import DataFrame.Internal.Column -import DataFrame.Internal.DataFrame (DataFrame (..)) -import DataFrame.Internal.Row -import DataFrame.Operations.Core - --- | Sort order taken as a parameter by the 'sortBy' function. -data SortOrder = Ascending | Descending deriving (Eq) - -{- | O(k log n) Sorts the dataframe by a given row. - -> sortBy Ascending ["Age"] df --} -sortBy :: - SortOrder -> - [T.Text] -> - DataFrame -> - DataFrame -sortBy order names df - | any (`notElem` columnNames df) names = - throw $ - ColumnNotFoundException - (T.pack $ show $ names L.\\ columnNames df) - "sortBy" - (columnNames df) - | otherwise = - let - indexes = sortedIndexes' (order == Ascending) (toRowVector names df) - in - df{columns = V.map (atIndicesStable indexes) (columns df)} diff --git a/src/DataFrame/Operations/Statistics.hs b/src/DataFrame/Operations/Statistics.hs --- a/src/DataFrame/Operations/Statistics.hs +++ b/src/DataFrame/Operations/Statistics.hs @@ -23,7 +23,13 @@ import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import DataFrame.Errors (DataFrameException (..)) import DataFrame.Internal.Column -import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn) +import DataFrame.Internal.DataFrame ( + DataFrame (..), + columnAsUnboxedVector, + empty, + getColumn, + ) +import DataFrame.Internal.Expression import DataFrame.Internal.Row (showValue, toAny) import DataFrame.Internal.Statistics import DataFrame.Internal.Types @@ -54,7 +60,7 @@ frequencies name df = let counts :: forall a. (Columnable a) => [(a, Int)] - counts = valueCounts name df + counts = valueCounts (Col @a name) df calculatePercentage cs k = toAny $ toPct2dp (fromIntegral k / fromIntegral (P.sum $ map snd cs)) initDf = empty @@ -79,28 +85,76 @@ Just ((UnboxedColumn (column :: VU.Vector a))) -> freqs column -- | Calculates the mean of a given column as a standalone value. -mean :: T.Text -> DataFrame -> Maybe Double -mean = applyStatistic mean' +mean :: + forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double +mean (Col name) df = case columnAsUnboxedVector @a name df of + Right xs -> mean' xs + Left e -> throw e +mean expr df = case interpret df expr of + Left e -> throw e + Right (TColumn col) -> case toUnboxedVector @a col of + Left e -> throw e + Right xs -> mean' xs -- | Calculates the median of a given column as a standalone value. -median :: T.Text -> DataFrame -> Maybe Double -median = applyStatistic median' +median :: + forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double +median (Col name) df = case columnAsUnboxedVector @a name df of + Right xs -> median' xs + Left e -> throw e +median expr df = case interpret df expr of + Left e -> throw e + Right (TColumn col) -> case toUnboxedVector @a col of + Left e -> throw e + Right xs -> median' xs -- | Calculates the standard deviation of a given column as a standalone value. -standardDeviation :: T.Text -> DataFrame -> Maybe Double -standardDeviation = applyStatistic (sqrt . variance') +standardDeviation :: + forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double +standardDeviation (Col name) df = case columnAsUnboxedVector @a name df of + Right xs -> (sqrt . variance') xs + Left e -> throw e +standardDeviation expr df = case interpret df expr of + Left e -> throw e + Right (TColumn col) -> case toUnboxedVector @a col of + Left e -> throw e + Right xs -> (sqrt . variance') xs -- | Calculates the skewness of a given column as a standalone value. -skewness :: T.Text -> DataFrame -> Maybe Double -skewness = applyStatistic skewness' +skewness :: + forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double +skewness (Col name) df = case columnAsUnboxedVector @a name df of + Right xs -> skewness' xs + Left e -> throw e +skewness expr df = case interpret df expr of + Left e -> throw e + Right (TColumn col) -> case toUnboxedVector @a col of + Left e -> throw e + Right xs -> skewness' xs -- | Calculates the variance of a given column as a standalone value. -variance :: T.Text -> DataFrame -> Maybe Double -variance = applyStatistic variance' +variance :: + forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double +variance (Col name) df = case columnAsUnboxedVector @a name df of + Right xs -> variance' xs + Left e -> throw e +variance expr df = case interpret df expr of + Left e -> throw e + Right (TColumn col) -> case toUnboxedVector @a col of + Left e -> throw e + Right xs -> variance' xs -- | Calculates the inter-quartile range of a given column as a standalone value. -interQuartileRange :: T.Text -> DataFrame -> Maybe Double -interQuartileRange = applyStatistic interQuartileRange' +interQuartileRange :: + forall a. (Columnable a, Real a, VU.Unbox a) => Expr a -> DataFrame -> Double +interQuartileRange (Col name) df = case columnAsUnboxedVector @a name df of + Right xs -> interQuartileRange' xs + Left e -> throw e +interQuartileRange expr df = case interpret df expr of + Left e -> throw e + Right (TColumn col) -> case toUnboxedVector @a col of + Left e -> throw e + Right xs -> interQuartileRange' xs -- | Calculates the Pearson's correlation coefficient between two given columns as a standalone value. correlation :: T.Text -> T.Text -> DataFrame -> Maybe Double @@ -126,18 +180,23 @@ -- | Calculates the sum of a given column as a standalone value. sum :: - forall a. (Columnable a, Num a, VU.Unbox a) => T.Text -> DataFrame -> Maybe a -sum name df = case getColumn name df of + forall a. (Columnable a, Num a) => Expr a -> DataFrame -> a +sum (Col name) df = case getColumn name df of Nothing -> throw $ ColumnNotFoundException name "sum" (M.keys $ columnIndices df) Just ((UnboxedColumn (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> Just $ VG.sum column - Nothing -> Nothing + Just Refl -> VG.sum column + Nothing -> 0 Just ((BoxedColumn (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> Just $ VG.sum column - Nothing -> Nothing + Just Refl -> VG.sum column + Nothing -> 0 Just ((OptionalColumn (column :: V.Vector (Maybe a')))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> Just $ VG.sum (VG.map (fromMaybe 0) column) - Nothing -> Nothing + Just Refl -> VG.sum (VG.map (fromMaybe 0) column) + Nothing -> 0 +sum expr df = case interpret df expr of + Left e -> throw e + Right (TColumn xs) -> case toVector @a @V.Vector xs of + Left e -> throw e + Right xs -> VG.sum xs applyStatistic :: (VU.Vector Double -> Double) -> T.Text -> DataFrame -> Maybe Double @@ -202,15 +261,15 @@ iqr = (-) <$> quartile3 <*> quartile1 in [ count - , mean name df + , mean' <$> _getColumnAsDouble name df , min' , quartile1 , median' , quartile3 , max' - , standardDeviation name df + , sqrt . variance' <$> _getColumnAsDouble name df , iqr - , skewness name df + , skewness' <$> _getColumnAsDouble name df ] -- | Round a @Double@ to Specified Precision diff --git a/src/DataFrame/Operations/Subset.hs b/src/DataFrame/Operations/Subset.hs --- a/src/DataFrame/Operations/Subset.hs +++ b/src/DataFrame/Operations/Subset.hs @@ -89,19 +89,33 @@ forall a. (Columnable a) => -- | Column to filter by - T.Text -> + Expr a -> -- | Filter condition (a -> Bool) -> -- | Dataframe to filter DataFrame -> DataFrame -filter filterColumnName condition df = case getColumn filterColumnName df of +filter (Col filterColumnName) condition df = case getColumn filterColumnName df of Nothing -> throw $ ColumnNotFoundException filterColumnName "filter" (M.keys $ columnIndices df) Just (BoxedColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df Just (OptionalColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df Just (UnboxedColumn (column :: VU.Vector b)) -> filterByVector filterColumnName column condition df +filter expr condition df = + let + (TColumn col) = case interpret @a df (normalize expr) of + Left e -> throw e + Right c -> c + indexes = case findIndices condition col of + Right ixs -> ixs + Left e -> throw e + c' = snd $ dataframeDimensions df + in + df + { columns = V.map (atIndicesStable indexes) (columns df) + , dataframeDimensions = (VU.length indexes, c') + } filterByVector :: forall a b v. @@ -131,7 +145,7 @@ > filterBy even "x" df -} -filterBy :: (Columnable a) => (a -> Bool) -> T.Text -> DataFrame -> DataFrame +filterBy :: (Columnable a) => (a -> Bool) -> Expr a -> DataFrame -> DataFrame filterBy = flip filter {- | O(k) filters the dataframe with a boolean expression. @@ -162,7 +176,7 @@ filterJust name df = case getColumn name df of Nothing -> throw $ ColumnNotFoundException name "filterJust" (M.keys $ columnIndices df) - Just column@(OptionalColumn (col :: V.Vector (Maybe a))) -> filter @(Maybe a) name isJust df & apply @(Maybe a) fromJust name + Just column@(OptionalColumn (col :: V.Vector (Maybe a))) -> filter (Col @(Maybe a) name) isJust df & apply @(Maybe a) fromJust name Just column -> df {- | O(k) returns all rows with `Nothing` in a give column. @@ -173,7 +187,7 @@ filterNothing name df = case getColumn name df of Nothing -> throw $ ColumnNotFoundException name "filterNothing" (M.keys $ columnIndices df) - Just (OptionalColumn (col :: V.Vector (Maybe a))) -> filter @(Maybe a) name isNothing df + Just (OptionalColumn (col :: V.Vector (Maybe a))) -> filter (Col @(Maybe a) name) isNothing df _ -> df {- | O(n * k) removes all rows with `Nothing` from the dataframe. @@ -182,6 +196,13 @@ -} filterAllJust :: DataFrame -> DataFrame filterAllJust df = foldr filterJust df (columnNames df) + +{- | O(n * k) keeps any row with a null value. + +> filterAllNothing df +-} +filterAllNothing :: DataFrame -> DataFrame +filterAllNothing df = foldr filterNothing df (columnNames df) {- | O(k) cuts the dataframe in a cube of size (a, b) where a is the length and b is the width. diff --git a/src/DataFrame/Operations/Transformations.hs b/src/DataFrame/Operations/Transformations.hs --- a/src/DataFrame/Operations/Transformations.hs +++ b/src/DataFrame/Operations/Transformations.hs @@ -1,4 +1,6 @@ +{-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE GADTs #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -68,6 +70,16 @@ Left e -> throw e Right (TColumn value) -> insertColumn name value df +deriveMany :: [NamedExpr] -> DataFrame -> DataFrame +deriveMany exprs df = + let + f (name, Wrap (expr :: Expr a)) d = + case interpret @a df expr of + Left e -> throw e + Right (TColumn value) -> insertColumn name value d + in + fold f exprs df + -- | O(k * n) Apply a function to given column names in a dataframe. applyMany :: (Columnable b, Columnable c) => @@ -162,11 +174,11 @@ impute :: forall b. (Columnable b) => - T.Text -> + Expr (Maybe b) -> b -> DataFrame -> DataFrame -impute columnName value df = case getColumn columnName df of +impute (Col columnName) value df = case getColumn columnName df of Nothing -> throw $ ColumnNotFoundException columnName "impute" (M.keys $ columnIndices df) Just (OptionalColumn _) -> case safeApply (fromMaybe value) columnName df of @@ -174,3 +186,4 @@ Left exception -> throw exception Right res -> res _ -> error $ "Cannot impute to a non-Empty column: " ++ T.unpack columnName +impute _ _ df = df diff --git a/src/DataFrame/Operations/Typing.hs b/src/DataFrame/Operations/Typing.hs --- a/src/DataFrame/Operations/Typing.hs +++ b/src/DataFrame/Operations/Typing.hs @@ -8,95 +8,112 @@ import qualified Data.Text as T import qualified Data.Vector as V -import qualified Data.Vector.Unboxed as VU -import Data.Either -import Data.Maybe +import Data.Maybe (fromMaybe) import Data.Time import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) -import DataFrame.Internal.Column (Column (..)) +import DataFrame.Internal.Column (Column (..), fromVector) import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Parsing import Type.Reflection (typeRep) -parseDefaults :: Int -> Bool -> String -> DataFrame -> DataFrame +type DateFormat = String + +parseDefaults :: Int -> Bool -> DateFormat -> DataFrame -> DataFrame parseDefaults n safeRead dateFormat df = df{columns = V.map (parseDefault n safeRead dateFormat) (columns df)} -parseDefault :: Int -> Bool -> String -> Column -> Column +parseDefault :: Int -> Bool -> DateFormat -> Column -> Column parseDefault n safeRead dateFormat (BoxedColumn (c :: V.Vector a)) = case (typeRep @a) `testEquality` (typeRep @T.Text) of Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of - Just Refl -> parseFromExamples n dateFormat safeRead (V.map T.pack c) + Just Refl -> parseFromExamples n safeRead dateFormat (V.map T.pack c) Nothing -> BoxedColumn c - Just Refl -> parseFromExamples n dateFormat safeRead c + Just Refl -> parseFromExamples n safeRead dateFormat c parseDefault n safeRead dateFormat (OptionalColumn (c :: V.Vector (Maybe a))) = case (typeRep @a) `testEquality` (typeRep @T.Text) of Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of - Just Refl -> parseFromExamples n dateFormat safeRead (V.map (T.pack . fromMaybe "") c) + Just Refl -> parseFromExamples n safeRead dateFormat (V.map (T.pack . fromMaybe "") c) Nothing -> BoxedColumn c - Just Refl -> parseFromExamples n dateFormat safeRead (V.map (fromMaybe "") c) + Just Refl -> parseFromExamples n safeRead dateFormat (V.map (fromMaybe "") c) parseDefault _ _ _ column = column -parseFromExamples :: Int -> String -> Bool -> V.Vector T.Text -> Column -parseFromExamples n dateFormat safeRead c - | V.any isJust (V.map readInt examples) - && equalNonEmpty (V.map readInt examples) (V.map readDouble examples) = - let safeVector = V.map readIntEither c - notAllParsed = V.any isLeft safeVector - in if safeRead && notAllParsed - then - if V.all (either isNullish (const False)) (V.filter isLeft safeVector) - then OptionalColumn (V.map (either (const Nothing) Just) safeVector) - else BoxedColumn safeVector - else - UnboxedColumn - (VU.generate (V.length c) (either undefined id . (safeVector V.!))) - | V.any isJust (V.map readDouble examples) = - let safeVector = V.map readDoubleEither c - notAllParsed = V.any isLeft safeVector - in if safeRead && notAllParsed - then - if V.all (either isNullish (const False)) (V.filter isLeft safeVector) - then OptionalColumn (V.map (either (const Nothing) Just) safeVector) - else BoxedColumn safeVector - else - UnboxedColumn - (VU.generate (V.length c) (either undefined id . (safeVector V.!))) - | V.any isJust (V.map (parseTimeOpt dateFormat) examples) = - let - -- failed parse should be Either, nullish should be Maybe - emptyToNothing' v = if isNullish v then Left v else Right v - parseTimeEither v = case parseTimeOpt dateFormat v of - Just v' -> Right v' - Nothing -> Left v - safeVector = V.map ((=<<) parseTimeEither . emptyToNothing') c - toMaybe (Left _) = Nothing - toMaybe (Right value) = Just value - lefts = V.filter isLeft safeVector - onlyNulls = (not (V.null lefts) && V.all (isNullish . fromLeft "non-null") lefts) - in - if safeRead - then - if onlyNulls - then OptionalColumn (V.map toMaybe safeVector) - else - if V.any isLeft safeVector - then BoxedColumn safeVector - else BoxedColumn (V.map (unsafeParseTime dateFormat) c) - else BoxedColumn (V.map (unsafeParseTime dateFormat) c) - | otherwise = - let - safeVector = V.map emptyToNothing c - hasNulls = V.any isNullish c - in - if safeRead && hasNulls then OptionalColumn safeVector else BoxedColumn c +parseFromExamples :: Int -> Bool -> DateFormat -> V.Vector T.Text -> Column +parseFromExamples n safeRead dateFormat cols = + let + converter = if safeRead then convertNullish else convertOnlyEmpty + examples = V.map converter (V.take n cols) + asMaybeText = V.map converter cols + in + case makeParsingAssumption dateFormat examples of + IntAssumption -> handleIntAssumption asMaybeText + DoubleAssumption -> handleDoubleAssumption asMaybeText + TextAssumption -> handleTextAssumption asMaybeText + DateAssumption -> handleDateAssumption dateFormat asMaybeText + NoAssumption -> handleNoAssumption dateFormat asMaybeText + +handleIntAssumption :: V.Vector (Maybe T.Text) -> Column +handleIntAssumption asMaybeText + | parsableAsInt = + maybe (fromVector asMaybeInt) fromVector (sequenceA asMaybeInt) + | parsableAsDouble = + maybe (fromVector asMaybeDouble) fromVector (sequenceA asMaybeDouble) + | otherwise = maybe (fromVector asMaybeText) fromVector (sequenceA asMaybeText) where - examples = V.take n c + asMaybeInt = V.map (>>= readInt) asMaybeText + asMaybeDouble = V.map (>>= readDouble) asMaybeText + parsableAsInt = + vecSameConstructor asMaybeText asMaybeInt + && vecSameConstructor asMaybeText asMaybeDouble + parsableAsDouble = vecSameConstructor asMaybeText asMaybeDouble -emptyToNothing :: T.Text -> Maybe T.Text -emptyToNothing v = if isNullish v then Nothing else Just v +handleDoubleAssumption :: V.Vector (Maybe T.Text) -> Column +handleDoubleAssumption asMaybeText + | parsableAsDouble = + maybe (fromVector asMaybeDouble) fromVector (sequenceA asMaybeDouble) + | otherwise = maybe (fromVector asMaybeText) fromVector (sequenceA asMaybeText) + where + asMaybeDouble = V.map (>>= readDouble) asMaybeText + parsableAsDouble = vecSameConstructor asMaybeText asMaybeDouble -parseTimeOpt :: String -> T.Text -> Maybe Day +handleDateAssumption :: DateFormat -> V.Vector (Maybe T.Text) -> Column +handleDateAssumption dateFormat asMaybeText + | parsableAsDate = + maybe (fromVector asMaybeDate) fromVector (sequenceA asMaybeDate) + | otherwise = maybe (fromVector asMaybeText) fromVector (sequenceA asMaybeText) + where + asMaybeDate = V.map (>>= parseTimeOpt dateFormat) asMaybeText + parsableAsDate = vecSameConstructor asMaybeText asMaybeDate + +handleTextAssumption :: V.Vector (Maybe T.Text) -> Column +handleTextAssumption asMaybeText = maybe (fromVector asMaybeText) fromVector (sequenceA asMaybeText) + +handleNoAssumption :: DateFormat -> V.Vector (Maybe T.Text) -> Column +handleNoAssumption dateFormat asMaybeText + -- No need to check for null values. If we are in this condition, that + -- means that the examples consisted only of null values, so we can + -- confidently know that this column must be an OptionalColumn + | V.all (== Nothing) asMaybeText = fromVector asMaybeText + | parsableAsInt = fromVector asMaybeInt + | parsableAsDouble = fromVector asMaybeDouble + | parsableAsDate = fromVector asMaybeDate + | otherwise = fromVector asMaybeText + where + asMaybeInt = V.map (>>= readInt) asMaybeText + asMaybeDouble = V.map (>>= readDouble) asMaybeText + asMaybeDate = V.map (>>= parseTimeOpt dateFormat) asMaybeText + parsableAsInt = + vecSameConstructor asMaybeText asMaybeInt + && vecSameConstructor asMaybeText asMaybeDouble + parsableAsDouble = vecSameConstructor asMaybeText asMaybeDouble + parsableAsDate = vecSameConstructor asMaybeText asMaybeDate + +convertNullish :: T.Text -> Maybe T.Text +convertNullish v = if isNullish v then Nothing else Just v + +convertOnlyEmpty :: T.Text -> Maybe T.Text +convertOnlyEmpty v = if v == "" then Nothing else Just v + +parseTimeOpt :: DateFormat -> T.Text -> Maybe Day parseTimeOpt dateFormat s = parseTimeM {- Accept leading/trailing whitespace -} True @@ -104,7 +121,7 @@ dateFormat (T.unpack s) -unsafeParseTime :: String -> T.Text -> Day +unsafeParseTime :: DateFormat -> T.Text -> Day unsafeParseTime dateFormat s = parseTimeOrError {- Accept leading/trailing whitespace -} True @@ -112,8 +129,44 @@ dateFormat (T.unpack s) -countNonEmpty :: V.Vector (Maybe a) -> Int -countNonEmpty xs = V.length (V.filter isJust xs) +hasNullValues :: (Eq a) => V.Vector (Maybe a) -> Bool +hasNullValues = V.any (== Nothing) -equalNonEmpty :: V.Vector (Maybe a) -> V.Vector (Maybe b) -> Bool -equalNonEmpty xs ys = countNonEmpty xs == countNonEmpty ys +vecSameConstructor :: V.Vector (Maybe a) -> V.Vector (Maybe b) -> Bool +vecSameConstructor xs ys = (V.length xs == V.length ys) && V.and (V.zipWith hasSameConstructor xs ys) + where + hasSameConstructor :: Maybe a -> Maybe b -> Bool + hasSameConstructor (Just _) (Just _) = True + hasSameConstructor Nothing Nothing = True + hasSameConstructor _ _ = False + +makeParsingAssumption :: + DateFormat -> V.Vector (Maybe T.Text) -> ParsingAssumption +makeParsingAssumption dateFormat asMaybeText + -- All the examples are "NA", "Null", "", so we can't make any shortcut + -- assumptions and just have to go the long way. + | V.all (== Nothing) asMaybeText = NoAssumption + -- After accounting for nulls, parsing for Ints and Doubles results in the + -- same corresponding positions of Justs and Nothings, so we assume + -- that the best way to parse is Int + | vecSameConstructor asMaybeText asMaybeInt + && vecSameConstructor asMaybeText asMaybeDouble = + IntAssumption + -- After accounting for nulls, the previous condition fails, so some (or none) can be parsed as Ints + -- and some can be parsed as Doubles, so we make the assumpotion of doubles. + | vecSameConstructor asMaybeText asMaybeDouble = DoubleAssumption + -- After accounting for nulls, parsing for Dates results in the same corresponding + -- positions of Justs and Nothings, so we assume that the best way to parse is Date. + | vecSameConstructor asMaybeText asMaybeDate = DateAssumption + | otherwise = TextAssumption + where + asMaybeInt = V.map (>>= readInt) asMaybeText + asMaybeDouble = V.map (>>= readDouble) asMaybeText + asMaybeDate = V.map (>>= parseTimeOpt dateFormat) asMaybeText + +data ParsingAssumption + = IntAssumption + | DoubleAssumption + | DateAssumption + | NoAssumption + | TextAssumption diff --git a/tests/Functions.hs b/tests/Functions.hs --- a/tests/Functions.hs +++ b/tests/Functions.hs @@ -6,14 +6,9 @@ import DataFrame.Functions ( col, generatePrograms, - lit, - pow, - relu, sanitize, ) -import DataFrame.Internal.Expression (add, mult) import Test.HUnit -import Prelude hiding (max, min) -- Test cases for the sanitize function sanitizeIdentifiers :: Test @@ -71,22 +66,8 @@ TestCase ( assertEqual "generatePrograms called with no existing programs" - [ col @Double "x" - , abs (col @Double "x") - , exp (mult (lit 0.5) (log (col @Double "x"))) - , log (add (lit 1.0) (col @Double "x")) - , exp (col @Double "x") - , sin (col @Double "x") - , cos (col @Double "x") - , relu (col @Double "x") - , signum (col @Double "x") - , pow 2 (col @Double "x") - , pow 3 (col @Double "x") - , pow 4 (col @Double "x") - , pow 5 (col @Double "x") - , pow 6 (col @Double "x") - ] - (generatePrograms [] [col "x"] [] []) + [col @Double "x"] + (generatePrograms True [] [col "x"] [] []) ) tests :: [Test] diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -22,179 +22,4946 @@ import qualified Operations.Filter import qualified Operations.GroupBy import qualified Operations.InsertColumn -import qualified Operations.Merge -import qualified Operations.ReadCsv -import qualified Operations.Sort -import qualified Operations.Statistics -import qualified Operations.Take -import qualified Parquet - -testData :: D.DataFrame -testData = - D.fromNamedColumns - [ ("test1", DI.fromList ([1 .. 26] :: [Int])) - , ("test2", DI.fromList ['a' .. 'z']) - ] - --- Dimensions -correctDimensions :: Test -correctDimensions = TestCase (assertEqual "should be (26, 2)" (26, 2) (D.dimensions testData)) - -emptyDataframeDimensions :: Test -emptyDataframeDimensions = TestCase (assertEqual "should be (0, 0)" (0, 0) (D.dimensions D.empty)) - -dimensionsTest :: [Test] -dimensionsTest = - [ TestLabel "dimensions_correctDimensions" correctDimensions - , TestLabel "dimensions_emptyDataframeDimensions" emptyDataframeDimensions - ] - --- parsing. -parseDate :: Test -parseDate = - let - expected = - DI.BoxedColumn - ( V.fromList - [fromGregorian 2020 02 14, fromGregorian 2021 02 14, fromGregorian 2022 02 14] - ) - actual = - D.parseDefault - 10 - True - "%Y-%m-%d" - (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-14", "2022-02-14"])) - in - TestCase (assertEqual "Correctly parses gregorian date" expected actual) - -parseInt :: Test -parseInt = - let - expected = - DI.UnboxedColumn - ( VU.fromList - [1 :: Int .. 5] - ) - actual = - D.parseDefault - 10 - True - "%Y-%m-%d" - (DI.fromVector (V.fromList ["1" :: T.Text, "2", "3", "4", "5"])) - in - TestCase (assertEqual "Correctly parses integers" expected actual) - -parseMaybeInt :: Test -parseMaybeInt = - let - expected = - DI.OptionalColumn - ( V.fromList - [Just (1 :: Int), Nothing, Just 3, Just 4, Just 5] - ) - actual = - D.parseDefault - 10 - True - "%Y-%m-%d" - (DI.fromVector (V.fromList ["1" :: T.Text, "N/A", "3", "4", "5"])) - in - TestCase (assertEqual "Correctly parses optional integers" expected actual) - -parseDouble :: Test -parseDouble = - let - expected = - DI.UnboxedColumn - ( VU.fromList - [1 :: Double, 2, 3, 4, 5] - ) - actual = - D.parseDefault - 10 - True - "%Y-%m-%d" - (DI.fromVector (V.fromList ["1" :: T.Text, "2.0", "3", "4", "5"])) - in - TestCase (assertEqual "Correctly parses doubles" expected actual) - -parseMaybeDouble :: Test -parseMaybeDouble = - let - expected = - DI.OptionalColumn - ( V.fromList - [Just 1 :: Maybe Double, Nothing, Just 3, Just 4, Just 5] - ) - actual = - D.parseDefault - 10 - True - "%Y-%m-%d" - (DI.fromVector (V.fromList ["1" :: T.Text, "N/A", "3.0", "4", "5"])) - in - TestCase (assertEqual "Correctly parses optional doubles" expected actual) - -incompleteDataParseEither :: Test -incompleteDataParseEither = - let - expected = - DI.BoxedColumn - ( V.fromList - [ Right $ fromGregorian 2020 02 14 - , Left ("2021-02-" :: T.Text) - , Right $ fromGregorian 2022 02 14 - ] - ) - actual = - D.parseDefault - 10 - True - "%Y-%m-%d" - (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-", "2022-02-14"])) - in - TestCase (assertEqual "Parses Either for gregorian date" expected actual) - -incompleteDataParseMaybe :: Test -incompleteDataParseMaybe = - let - expected = - DI.OptionalColumn - ( V.fromList - [Just $ fromGregorian 2020 02 14, Nothing, Just $ fromGregorian 2022 02 14] - ) - actual = - D.parseDefault - 10 - True - "%Y-%m-%d" - (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "", "2022-02-14"])) - in - TestCase - (assertEqual "Parses Maybe for gregorian date with null/empty" expected actual) - -parseTests :: [Test] -parseTests = - [ TestLabel "parseDate" parseDate - , TestLabel "incompleteDataParseMaybe" incompleteDataParseMaybe - , TestLabel "incompleteDataParseEither" incompleteDataParseEither - , TestLabel "parseInt" parseInt - , TestLabel "parseMaybeInt" parseMaybeInt - , TestLabel "parseDouble" parseDouble - , TestLabel "parseMaybeDouble" parseMaybeDouble - ] - -tests :: Test -tests = - TestList $ - dimensionsTest - ++ Operations.Aggregations.tests - ++ Operations.Apply.tests - ++ Operations.Core.tests - ++ Operations.Derive.tests - ++ Operations.Filter.tests - ++ Operations.GroupBy.tests - ++ Operations.InsertColumn.tests +import qualified Operations.Join +import qualified Operations.Merge +import qualified Operations.ReadCsv +import qualified Operations.Sort +import qualified Operations.Statistics +import qualified Operations.Take +import qualified Parquet + +testData :: D.DataFrame +testData = + D.fromNamedColumns + [ ("test1", DI.fromList ([1 .. 26] :: [Int])) + , ("test2", DI.fromList ['a' .. 'z']) + ] + +-- Dimensions +correctDimensions :: Test +correctDimensions = TestCase (assertEqual "should be (26, 2)" (26, 2) (D.dimensions testData)) + +emptyDataframeDimensions :: Test +emptyDataframeDimensions = TestCase (assertEqual "should be (0, 0)" (0, 0) (D.dimensions D.empty)) + +dimensionsTest :: [Test] +dimensionsTest = + [ TestLabel "dimensions_correctDimensions" correctDimensions + , TestLabel "dimensions_emptyDataframeDimensions" emptyDataframeDimensions + ] + +-- PARSING TESTS +------- 1. SIMPLE CASES +parseInts :: Test +parseInts = + let afterParse :: [Int] + afterParse = [1 .. 50] + beforeParse :: [T.Text] + beforeParse = T.pack . show <$> [1 .. 50] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as UnboxedColumn of Ints" + expected + actual + ) + +parseDoubles :: Test +parseDoubles = + let afterParse :: [Double] + afterParse = [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] + beforeParse :: [T.Text] + beforeParse = + T.pack . show + <$> [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Doubles without missing values as UnboxedColumn of Doubles" + expected + actual + ) + +parseDates :: Test +parseDates = + let afterParse :: [Day] + afterParse = + [ fromGregorian 2020 02 12 + , fromGregorian 2020 02 13 + , fromGregorian 2020 02 14 + , fromGregorian 2020 02 15 + , fromGregorian 2020 02 16 + , fromGregorian 2020 02 17 + , fromGregorian 2020 02 18 + , fromGregorian 2020 02 19 + , fromGregorian 2020 02 20 + , fromGregorian 2020 02 21 + , fromGregorian 2020 02 22 + , fromGregorian 2020 02 23 + , fromGregorian 2020 02 24 + , fromGregorian 2020 02 25 + , fromGregorian 2020 02 26 + ] + beforeParse :: [T.Text] + beforeParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as BoxedColumn of Days" + expected + actual + ) + +parseTexts :: Test +parseTexts = + let afterParse :: [T.Text] + afterParse = + [ "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "Surrender now or prepare to fight!" + , "Meowth, that's right!" + ] + beforeParse :: [T.Text] + beforeParse = + [ "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "Surrender now or prepare to fight!" + , "Meowth, that's right!" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Text without missing values as BoxedColumn of Text" + expected + actual + ) + +--- 2. COMBINATION CASES +parseIntsAndDoublesAsDoubles :: Test +parseIntsAndDoublesAsDoubles = + let afterParse :: [Double] + afterParse = + [ 1 + , 2 + , 3 + , 4 + , 5 + , 6 + , 7 + , 8 + , 9 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 26 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 35 + , 36 + , 37 + , 38 + , 39 + , 40 + , 41 + , 42 + , 43 + , 44 + , 45 + , 46 + , 47 + , 48 + , 49 + , 50 + , 1.0 + , 2.0 + , 3.0 + , 4.0 + , 5.0 + , 6.0 + , 7.0 + , 8.0 + , 9.0 + , 10.0 + , 11.0 + , 12.0 + , 13.0 + , 14.0 + , 15.0 + , 16.0 + , 17.0 + , 18.0 + , 19.0 + , 20.0 + , 21.0 + , 22.0 + , 23.0 + , 24.0 + , 25.0 + , 26.0 + , 27.0 + , 28.0 + , 29.0 + , 30.0 + , 31.0 + , 32.0 + , 33.0 + , 34.0 + , 35.0 + , 36.0 + , 37.0 + , 38.0 + , 39.0 + , 40.0 + , 41.0 + , 42.0 + , 43.0 + , 44.0 + , 45.0 + , 46.0 + , 47.0 + , 48.0 + , 49.0 + , 50.0 + , 3.14 + , 2.22 + , 8.55 + , 23.3 + , 12.22222235049450945049504950 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles" + expected + actual + ) + +parseIntsAndDatesAsTexts :: Test +parseIntsAndDatesAsTexts = + let afterParse :: [T.Text] + afterParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Dates as BoxedColumn of Texts" + expected + actual + ) + +parseTextsAndDoublesAsTexts :: Test +parseTextsAndDoublesAsTexts = + let afterParse :: [T.Text] + afterParse = + [ "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + beforeParse :: [T.Text] + beforeParse = + [ "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Texts and Doubles as BoxedColumn of Texts" + expected + actual + ) + +parseDatesAndTextsAsTexts :: Test +parseDatesAndTextsAsTexts = + let afterParse :: [T.Text] + afterParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + , "Jessie" + , "James" + , "Meowth" + ] + beforeParse :: [T.Text] + beforeParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + , "Jessie" + , "James" + , "Meowth" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Dates and Texts as BoxedColumn of Texts" + expected + actual + ) + +-- 3A. PARSING WITH SAFEREAD OFF + +parseIntsWithoutSafeRead :: Test +parseIntsWithoutSafeRead = + let afterParse :: [Int] + afterParse = + [ 1 + , 2 + , 3 + , 4 + , 5 + , 6 + , 7 + , 8 + , 9 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 26 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 35 + , 36 + , 37 + , 38 + , 39 + , 40 + , 41 + , 42 + , 43 + , 44 + , 45 + , 46 + , 47 + , 48 + , 49 + , 50 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as UnboxedColumn of Ints, when safeRead is off" + expected + actual + ) + +parseDoublesWithoutSafeRead :: Test +parseDoublesWithoutSafeRead = + let afterParse :: [Double] + afterParse = + [ 1.0 + , 2.0 + , 3.0 + , 4.0 + , 5.0 + , 6.0 + , 7.0 + , 8.0 + , 9.0 + , 10.0 + , 11.0 + , 12.0 + , 13.0 + , 14.0 + , 15.0 + , 16.0 + , 17.0 + , 18.0 + , 19.0 + , 20.0 + , 21.0 + , 22.0 + , 23.0 + , 24.0 + , 25.0 + , 26.0 + , 27.0 + , 28.0 + , 29.0 + , 30.0 + , 31.0 + , 32.0 + , 33.0 + , 34.0 + , 35.0 + , 36.0 + , 37.0 + , 38.0 + , 39.0 + , 40.0 + , 41.0 + , 42.0 + , 43.0 + , 44.0 + , 45.0 + , 46.0 + , 47.0 + , 48.0 + , 49.0 + , 50.0 + , 3.14 + , 2.22 + , 8.55 + , 23.3 + , 12.22222235049450945049504950 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Doubles without missing values as UnboxedColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseDatesWithoutSafeRead :: Test +parseDatesWithoutSafeRead = + let afterParse :: [Day] + afterParse = + [ fromGregorian 2020 02 12 + , fromGregorian 2020 02 13 + , fromGregorian 2020 02 14 + , fromGregorian 2020 02 15 + , fromGregorian 2020 02 16 + , fromGregorian 2020 02 17 + , fromGregorian 2020 02 18 + , fromGregorian 2020 02 19 + , fromGregorian 2020 02 20 + , fromGregorian 2020 02 21 + , fromGregorian 2020 02 22 + , fromGregorian 2020 02 23 + , fromGregorian 2020 02 24 + , fromGregorian 2020 02 25 + , fromGregorian 2020 02 26 + ] + beforeParse :: [T.Text] + beforeParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as BoxedColumn of Days" + expected + actual + ) + +parseTextsWithoutSafeRead :: Test +parseTextsWithoutSafeRead = + let afterParse :: [T.Text] + afterParse = + [ "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "Surrender now or prepare to fight!" + , "Meowth, that's right!" + ] + beforeParse :: [T.Text] + beforeParse = + [ "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "Surrender now or prepare to fight!" + , "Meowth, that's right!" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Text without missing values as BoxedColumn of Text" + expected + actual + ) + +parseIntsAndEmptyStringsWithoutSafeRead :: Test +parseIntsAndEmptyStringsWithoutSafeRead = + let afterParse :: [Maybe Int] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just 1 + , Just 2 + , Just 3 + , Just 4 + , Just 5 + , Just 6 + , Just 7 + , Just 8 + , Just 9 + , Just 10 + , Just 11 + , Just 12 + , Just 13 + , Just 14 + , Just 15 + , Just 16 + , Just 17 + , Just 18 + , Just 19 + , Just 20 + , Just 21 + , Just 22 + , Just 23 + , Just 24 + , Just 25 + , Just 26 + , Just 27 + , Just 28 + , Just 29 + , Just 30 + , Just 31 + , Just 32 + , Just 33 + , Just 34 + , Just 35 + , Just 36 + , Just 37 + , Just 38 + , Just 39 + , Just 40 + , Just 41 + , Just 42 + , Just 43 + , Just 44 + , Just 45 + , Just 46 + , Just 47 + , Just 48 + , Just 49 + , Just 50 + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "" + , "" + , "" + , "" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is off" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsWithoutSafeRead :: Test +parseIntsAndDoublesAndEmptyStringsWithoutSafeRead = + let afterParse :: [Maybe Double] + afterParse = + [ Just 1.0 + , Just 2.0 + , Just 3.0 + , Just 4.0 + , Just 5.0 + , Just 6.0 + , Just 7.0 + , Just 8.0 + , Just 9.0 + , Just 10.0 + , Nothing + , Just 11.0 + , Just 12.0 + , Just 13.0 + , Just 14.0 + , Just 15.0 + , Just 16.0 + , Just 17.0 + , Just 18.0 + , Just 19.0 + , Just 20.0 + , Nothing + , Just 21.0 + , Just 22.0 + , Just 23.0 + , Just 24.0 + , Just 25.0 + , Just 26.0 + , Just 27.0 + , Just 28.0 + , Just 29.0 + , Just 30.0 + , Nothing + , Just 31.0 + , Just 32.0 + , Just 33.0 + , Just 34.0 + , Just 35.0 + , Just 36.0 + , Just 37.0 + , Just 38.0 + , Just 39.0 + , Just 40.0 + , Nothing + , Just 41.0 + , Just 42.0 + , Just 43.0 + , Just 44.0 + , Just 45.0 + , Just 46.0 + , Just 47.0 + , Just 48.0 + , Just 49.0 + , Just 50.0 + , Nothing + , Just 3.14 + , Just 2.22 + , Just 8.55 + , Just 23.3 + , Just 12.22222235049451 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseDatesAndEmptyStringsWithoutSafeRead :: Test +parseDatesAndEmptyStringsWithoutSafeRead = + let afterParse :: [Maybe Day] + afterParse = + [ Just $ fromGregorian 2020 02 12 + , Just $ fromGregorian 2020 02 13 + , Just $ fromGregorian 2020 02 14 + , Nothing + , Just $ fromGregorian 2020 02 15 + , Just $ fromGregorian 2020 02 16 + , Just $ fromGregorian 2020 02 17 + , Nothing + , Just $ fromGregorian 2020 02 18 + , Just $ fromGregorian 2020 02 19 + , Just $ fromGregorian 2020 02 20 + , Nothing + , Just $ fromGregorian 2020 02 21 + , Just $ fromGregorian 2020 02 22 + , Just $ fromGregorian 2020 02 23 + , Nothing + , Just $ fromGregorian 2020 02 24 + , Just $ fromGregorian 2020 02 25 + , Just $ fromGregorian 2020 02 26 + , Nothing + ] + beforeParse :: [T.Text] + beforeParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + , "" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead off" + expected + actual + ) + +parseTextsAndEmptyStringsWithoutSafeRead :: Test +parseTextsAndEmptyStringsWithoutSafeRead = + let afterParse :: [Maybe T.Text] + afterParse = + [ Nothing + , Just "To" + , Just "protect" + , Just "the" + , Just "world" + , Just "from" + , Just "devastation" + , Nothing + , Just "To" + , Just "unite" + , Just "all" + , Just "people" + , Just "within" + , Just "our" + , Just "nation" + , Nothing + , Just "To" + , Just "denounce" + , Just "the" + , Just "evils" + , Just "of" + , Just "truth" + , Just "and" + , Just "love" + , Nothing + , Just "To" + , Just "extend" + , Just "our" + , Just "reach" + , Just "to" + , Just "the" + , Just "stars" + , Just "above" + , Nothing + , Just "JESSIE!" + , Just "JAMES!" + , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , Nothing + , Just "Surrender now or prepare to fight!" + , Nothing + , Just "Meowth, that's right!" + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "" + , "Surrender now or prepare to fight!" + , "" + , "Meowth, that's right!" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" + expected + actual + ) + +parseIntsAndNullishStringsWithoutSafeRead :: Test +parseIntsAndNullishStringsWithoutSafeRead = + let afterParse :: [T.Text] + afterParse = + [ "N/A" + , "N/A" + , "N/A" + , "N/A" + , "N/A" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + beforeParse :: [T.Text] + beforeParse = + [ "N/A" + , "N/A" + , "N/A" + , "N/A" + , "N/A" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values as BoxedColumn of Texts, when safeRead is off" + expected + actual + ) + +parseIntsAndDoublesAndNullishStringsWithoutSafeRead :: Test +parseIntsAndDoublesAndNullishStringsWithoutSafeRead = + let afterParse :: [T.Text] + afterParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "Nothing" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "N/A" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "NULL" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "null" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "NAN" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "Nothing" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "N/A" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "NULL" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "null" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "NAN" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseIntsAndNullishAndEmptyStringsWithoutSafeRead :: Test +parseIntsAndNullishAndEmptyStringsWithoutSafeRead = + let afterParse :: [Maybe T.Text] + afterParse = + [ Just "N/A" + , Just "N/A" + , Just "N/A" + , Just "N/A" + , Just "N/A" + , Nothing + , Just "1" + , Just "2" + , Just "3" + , Just "4" + , Just "5" + , Just "6" + , Just "7" + , Just "8" + , Just "9" + , Just "10" + , Nothing + , Just "11" + , Just "12" + , Just "13" + , Just "14" + , Just "15" + , Just "16" + , Just "17" + , Just "18" + , Just "19" + , Just "20" + , Nothing + , Just "21" + , Just "22" + , Just "23" + , Just "24" + , Just "25" + , Just "26" + , Just "27" + , Just "28" + , Just "29" + , Just "30" + , Nothing + , Just "31" + , Just "32" + , Just "33" + , Just "34" + , Just "35" + , Just "36" + , Just "37" + , Just "38" + , Just "39" + , Just "40" + , Nothing + , Just "41" + , Just "42" + , Just "43" + , Just "44" + , Just "45" + , Just "46" + , Just "47" + , Just "48" + , Just "49" + , Just "50" + , Nothing + ] + beforeParse :: [T.Text] + beforeParse = + [ "N/A" + , "N/A" + , "N/A" + , "N/A" + , "N/A" + , "" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + , "" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Texts, when safeRead is off" + expected + actual + ) + +parseTextsAndEmptyAndNullishStringsWithoutSafeRead :: Test +parseTextsAndEmptyAndNullishStringsWithoutSafeRead = + let afterParse :: [Maybe T.Text] + afterParse = + [ Nothing + , Just "To" + , Just "protect" + , Just "the" + , Just "world" + , Just "from" + , Just "devastation" + , Nothing + , Just "To" + , Just "unite" + , Just "all" + , Just "people" + , Just "within" + , Just "our" + , Just "nation" + , Nothing + , Just "To" + , Just "denounce" + , Just "the" + , Just "evils" + , Just "of" + , Just "truth" + , Just "and" + , Just "love" + , Nothing + , Just "To" + , Just "extend" + , Just "our" + , Just "reach" + , Just "to" + , Just "the" + , Just "stars" + , Just "above" + , Nothing + , Just "JESSIE!" + , Just "JAMES!" + , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , Nothing + , Just "Surrender now or prepare to fight!" + , Nothing + , Just "Meowth, that's right!" + , Just "NaN" + , Just "Nothing" + , Just "N/A" + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "" + , "Surrender now or prepare to fight!" + , "" + , "Meowth, that's right!" + , "NaN" + , "Nothing" + , "N/A" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" + expected + actual + ) + +-- 3B. PARSING WITH SAFEREAD ON +parseIntsAndEmptyStringsWithSafeRead :: Test +parseIntsAndEmptyStringsWithSafeRead = + let afterParse :: [Maybe Int] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just 1 + , Just 2 + , Just 3 + , Just 4 + , Just 5 + , Just 6 + , Just 7 + , Just 8 + , Just 9 + , Just 10 + , Just 11 + , Just 12 + , Just 13 + , Just 14 + , Just 15 + , Just 16 + , Just 17 + , Just 18 + , Just 19 + , Just 20 + , Just 21 + , Just 22 + , Just 23 + , Just 24 + , Just 25 + , Just 26 + , Just 27 + , Just 28 + , Just 29 + , Just 30 + , Just 31 + , Just 32 + , Just 33 + , Just 34 + , Just 35 + , Just 36 + , Just 37 + , Just 38 + , Just 39 + , Just 40 + , Just 41 + , Just 42 + , Just 43 + , Just 44 + , Just 45 + , Just 46 + , Just 47 + , Just 48 + , Just 49 + , Just 50 + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "" + , "" + , "" + , "" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is on" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsWithSafeRead :: Test +parseIntsAndDoublesAndEmptyStringsWithSafeRead = + let afterParse :: [Maybe Double] + afterParse = + [ Just 1.0 + , Just 2.0 + , Just 3.0 + , Just 4.0 + , Just 5.0 + , Just 6.0 + , Just 7.0 + , Just 8.0 + , Just 9.0 + , Just 10.0 + , Nothing + , Just 11.0 + , Just 12.0 + , Just 13.0 + , Just 14.0 + , Just 15.0 + , Just 16.0 + , Just 17.0 + , Just 18.0 + , Just 19.0 + , Just 20.0 + , Nothing + , Just 21.0 + , Just 22.0 + , Just 23.0 + , Just 24.0 + , Just 25.0 + , Just 26.0 + , Just 27.0 + , Just 28.0 + , Just 29.0 + , Just 30.0 + , Nothing + , Just 31.0 + , Just 32.0 + , Just 33.0 + , Just 34.0 + , Just 35.0 + , Just 36.0 + , Just 37.0 + , Just 38.0 + , Just 39.0 + , Just 40.0 + , Nothing + , Just 41.0 + , Just 42.0 + , Just 43.0 + , Just 44.0 + , Just 45.0 + , Just 46.0 + , Just 47.0 + , Just 48.0 + , Just 49.0 + , Just 50.0 + , Nothing + , Just 3.14 + , Just 2.22 + , Just 8.55 + , Just 23.3 + , Just 12.22222235049451 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is on" + expected + actual + ) + +parseDatesAndEmptyStringsWithSafeRead :: Test +parseDatesAndEmptyStringsWithSafeRead = + let afterParse :: [Maybe Day] + afterParse = + [ Just $ fromGregorian 2020 02 12 + , Just $ fromGregorian 2020 02 13 + , Just $ fromGregorian 2020 02 14 + , Nothing + , Just $ fromGregorian 2020 02 15 + , Just $ fromGregorian 2020 02 16 + , Just $ fromGregorian 2020 02 17 + , Nothing + , Just $ fromGregorian 2020 02 18 + , Just $ fromGregorian 2020 02 19 + , Just $ fromGregorian 2020 02 20 + , Nothing + , Just $ fromGregorian 2020 02 21 + , Just $ fromGregorian 2020 02 22 + , Just $ fromGregorian 2020 02 23 + , Nothing + , Just $ fromGregorian 2020 02 24 + , Just $ fromGregorian 2020 02 25 + , Just $ fromGregorian 2020 02 26 + , Nothing + ] + beforeParse :: [T.Text] + beforeParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + , "" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead on" + expected + actual + ) + +parseTextsAndEmptyStringsWithSafeRead :: Test +parseTextsAndEmptyStringsWithSafeRead = + let afterParse :: [Maybe T.Text] + afterParse = + [ Nothing + , Just "To" + , Just "protect" + , Just "the" + , Just "world" + , Just "from" + , Just "devastation" + , Nothing + , Just "To" + , Just "unite" + , Just "all" + , Just "people" + , Just "within" + , Just "our" + , Just "nation" + , Nothing + , Just "To" + , Just "denounce" + , Just "the" + , Just "evils" + , Just "of" + , Just "truth" + , Just "and" + , Just "love" + , Nothing + , Just "To" + , Just "extend" + , Just "our" + , Just "reach" + , Just "to" + , Just "the" + , Just "stars" + , Just "above" + , Nothing + , Just "JESSIE!" + , Just "JAMES!" + , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , Nothing + , Just "Surrender now or prepare to fight!" + , Nothing + , Just "Meowth, that's right!" + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "" + , "Surrender now or prepare to fight!" + , "" + , "Meowth, that's right!" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead on" + expected + actual + ) + +parseIntsAndNullishStringsWithSafeRead :: Test +parseIntsAndNullishStringsWithSafeRead = + let afterParse :: [Maybe Int] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just 1 + , Just 2 + , Just 3 + , Just 4 + , Just 5 + , Just 6 + , Just 7 + , Just 8 + , Just 9 + , Just 10 + , Just 11 + , Just 12 + , Just 13 + , Just 14 + , Just 15 + , Just 16 + , Just 17 + , Just 18 + , Just 19 + , Just 20 + , Just 21 + , Just 22 + , Just 23 + , Just 24 + , Just 25 + , Just 26 + , Just 27 + , Just 28 + , Just 29 + , Just 30 + , Just 31 + , Just 32 + , Just 33 + , Just 34 + , Just 35 + , Just 36 + , Just 37 + , Just 38 + , Just 39 + , Just 40 + , Just 41 + , Just 42 + , Just 43 + , Just 44 + , Just 45 + , Just 46 + , Just 47 + , Just 48 + , Just 49 + , Just 50 + ] + beforeParse :: [T.Text] + beforeParse = + [ "N/A" + , "N/A" + , "N/A" + , "N/A" + , "N/A" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values as OptionalColumn of Ints, when safeRead is on" + expected + actual + ) + +parseIntsAndDoublesAndNullishStringsWithSafeRead :: Test +parseIntsAndDoublesAndNullishStringsWithSafeRead = + let afterParse :: [Maybe Double] + afterParse = + [ Just 1.0 + , Just 2.0 + , Just 3.0 + , Just 4.0 + , Just 5.0 + , Just 6.0 + , Just 7.0 + , Just 8.0 + , Just 9.0 + , Just 10.0 + , Nothing + , Just 11.0 + , Just 12.0 + , Just 13.0 + , Just 14.0 + , Just 15.0 + , Just 16.0 + , Just 17.0 + , Just 18.0 + , Just 19.0 + , Just 20.0 + , Nothing + , Just 21.0 + , Just 22.0 + , Just 23.0 + , Just 24.0 + , Just 25.0 + , Just 26.0 + , Just 27.0 + , Just 28.0 + , Just 29.0 + , Just 30.0 + , Nothing + , Just 31.0 + , Just 32.0 + , Just 33.0 + , Just 34.0 + , Just 35.0 + , Just 36.0 + , Just 37.0 + , Just 38.0 + , Just 39.0 + , Just 40.0 + , Nothing + , Just 41.0 + , Just 42.0 + , Just 43.0 + , Just 44.0 + , Just 45.0 + , Just 46.0 + , Just 47.0 + , Just 48.0 + , Just 49.0 + , Just 50.0 + , Nothing + , Just 3.14 + , Just 2.22 + , Just 8.55 + , Just 23.3 + , Just 12.03 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "Nothing" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "N/A" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "NULL" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "null" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "NAN" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.03" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseIntsAndNullishAndEmptyStringsWithSafeRead :: Test +parseIntsAndNullishAndEmptyStringsWithSafeRead = + let afterParse :: [Maybe Int] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just 1 + , Just 2 + , Just 3 + , Just 4 + , Just 5 + , Just 6 + , Just 7 + , Just 8 + , Just 9 + , Just 10 + , Nothing + , Just 11 + , Just 12 + , Just 13 + , Just 14 + , Just 15 + , Just 16 + , Just 17 + , Just 18 + , Just 19 + , Just 20 + , Nothing + , Just 21 + , Just 22 + , Just 23 + , Just 24 + , Just 25 + , Just 26 + , Just 27 + , Just 28 + , Just 29 + , Just 30 + , Nothing + , Just 31 + , Just 32 + , Just 33 + , Just 34 + , Just 35 + , Just 36 + , Just 37 + , Just 38 + , Just 39 + , Just 40 + , Nothing + , Just 41 + , Just 42 + , Just 43 + , Just 44 + , Just 45 + , Just 46 + , Just 47 + , Just 48 + , Just 49 + , Just 50 + , Nothing + ] + beforeParse :: [T.Text] + beforeParse = + [ "N/A" + , "N/A" + , "N/A" + , "N/A" + , "Nothing" + , "" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + , "" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Ints, when safeRead is on" + expected + actual + ) + +parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead :: Test +parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead = + let afterParse :: [Maybe Double] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just 1 + , Just 2 + , Just 3 + , Just 4 + , Just 5 + , Just 6 + , Just 7 + , Just 8 + , Just 9 + , Just 10 + , Nothing + , Just 11 + , Just 12 + , Just 13 + , Just 14 + , Just 15 + , Just 16 + , Just 17 + , Just 18 + , Just 19 + , Just 20 + , Nothing + , Just 21 + , Just 22 + , Just 23 + , Just 24 + , Just 25 + , Just 26 + , Just 27 + , Just 28 + , Just 29 + , Just 30 + , Nothing + , Just 3.14 + ] + beforeParse :: [T.Text] + beforeParse = + [ "N/A" + , "N/A" + , "N/A" + , "N/A" + , "Nothing" + , "" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "" + , "3.14" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints and Doubles with nullish values AND empty strings as OptionalColumn of Doubles, when safeRead is on" + expected + actual + ) + +parseTextsAndEmptyAndNullishStringsWithSafeRead :: Test +parseTextsAndEmptyAndNullishStringsWithSafeRead = + let afterParse :: [Maybe T.Text] + afterParse = + [ Nothing + , Just "To" + , Just "protect" + , Just "the" + , Just "world" + , Just "from" + , Just "devastation" + , Nothing + , Just "To" + , Just "unite" + , Just "all" + , Just "people" + , Just "within" + , Just "our" + , Just "nation" + , Nothing + , Just "To" + , Just "denounce" + , Just "the" + , Just "evils" + , Just "of" + , Just "truth" + , Just "and" + , Just "love" + , Nothing + , Just "To" + , Just "extend" + , Just "our" + , Just "reach" + , Just "to" + , Just "the" + , Just "stars" + , Just "above" + , Nothing + , Just "JESSIE!" + , Just "JAMES!" + , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , Nothing + , Just "Surrender now or prepare to fight!" + , Nothing + , Just "Meowth, that's right!" + , Nothing + , Nothing + , Nothing + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "To" + , "protect" + , "the" + , "world" + , "from" + , "devastation" + , "" + , "To" + , "unite" + , "all" + , "people" + , "within" + , "our" + , "nation" + , "" + , "To" + , "denounce" + , "the" + , "evils" + , "of" + , "truth" + , "and" + , "love" + , "" + , "To" + , "extend" + , "our" + , "reach" + , "to" + , "the" + , "stars" + , "above" + , "" + , "JESSIE!" + , "JAMES!" + , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" + , "" + , "Surrender now or prepare to fight!" + , "" + , "Meowth, that's right!" + , "NaN" + , "Nothing" + , "N/A" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" + expected + actual + ) + +-- 4. PARSING SHOULD NOT DEPEND ON THE NUMBER OF EXAMPLES. +parseIntsWithOneExample :: Test +parseIntsWithOneExample = + let afterParse :: [Int] + afterParse = + [ 1 + , 2 + , 3 + , 4 + , 5 + , 6 + , 7 + , 8 + , 9 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 26 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 35 + , 36 + , 37 + , 38 + , 39 + , 40 + , 41 + , 42 + , 43 + , 44 + , 45 + , 46 + , 47 + , 48 + , 49 + , 50 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as UnboxedColumn of Ints with only one example" + expected + actual + ) + +parseIntsWithTwentyFiveExamples :: Test +parseIntsWithTwentyFiveExamples = + let afterParse :: [Int] + afterParse = + [ 1 + , 2 + , 3 + , 4 + , 5 + , 6 + , 7 + , 8 + , 9 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 26 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 35 + , 36 + , 37 + , 38 + , 39 + , 40 + , 41 + , 42 + , 43 + , 44 + , 45 + , 46 + , 47 + , 48 + , 49 + , 50 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 25 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as UnboxedColumn of Ints with some examples" + expected + actual + ) + +parseIntsWithFortyNineExamples :: Test +parseIntsWithFortyNineExamples = + let afterParse :: [Int] + afterParse = + [ 1 + , 2 + , 3 + , 4 + , 5 + , 6 + , 7 + , 8 + , 9 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 26 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 35 + , 36 + , 37 + , 38 + , 39 + , 40 + , 41 + , 42 + , 43 + , 44 + , 45 + , 46 + , 47 + , 48 + , 49 + , 50 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as UnboxedColumn of Ints with many examples" + expected + actual + ) + +parseDatesWithOneExample :: Test +parseDatesWithOneExample = + let afterParse :: [Day] + afterParse = + [ fromGregorian 2020 02 12 + , fromGregorian 2020 02 13 + , fromGregorian 2020 02 14 + , fromGregorian 2020 02 15 + , fromGregorian 2020 02 16 + , fromGregorian 2020 02 17 + , fromGregorian 2020 02 18 + , fromGregorian 2020 02 19 + , fromGregorian 2020 02 20 + , fromGregorian 2020 02 21 + , fromGregorian 2020 02 22 + , fromGregorian 2020 02 23 + , fromGregorian 2020 02 24 + , fromGregorian 2020 02 25 + , fromGregorian 2020 02 26 + ] + beforeParse :: [T.Text] + beforeParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as BoxedColumn of Days with only one example" + expected + actual + ) + +parseDatesWithFifteenExamples :: Test +parseDatesWithFifteenExamples = + let afterParse :: [Day] + afterParse = + [ fromGregorian 2020 02 12 + , fromGregorian 2020 02 13 + , fromGregorian 2020 02 14 + , fromGregorian 2020 02 15 + , fromGregorian 2020 02 16 + , fromGregorian 2020 02 17 + , fromGregorian 2020 02 18 + , fromGregorian 2020 02 19 + , fromGregorian 2020 02 20 + , fromGregorian 2020 02 21 + , fromGregorian 2020 02 22 + , fromGregorian 2020 02 23 + , fromGregorian 2020 02 24 + , fromGregorian 2020 02 25 + , fromGregorian 2020 02 26 + ] + beforeParse :: [T.Text] + beforeParse = + [ "2020-02-12" + , "2020-02-13" + , "2020-02-14" + , "2020-02-15" + , "2020-02-16" + , "2020-02-17" + , "2020-02-18" + , "2020-02-19" + , "2020-02-20" + , "2020-02-21" + , "2020-02-22" + , "2020-02-23" + , "2020-02-24" + , "2020-02-25" + , "2020-02-26" + ] + expected = DI.BoxedColumn $ V.fromList afterParse + actual = D.parseDefault 15 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as BoxedColumn of Days with many examples" + expected + actual + ) + +parseIntsAndDoublesAsDoublesWithOneExample :: Test +parseIntsAndDoublesAsDoublesWithOneExample = + let afterParse :: [Double] + afterParse = + [ 1 + , 2 + , 3 + , 4 + , 5 + , 6 + , 7 + , 8 + , 9 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 26 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 35 + , 36 + , 37 + , 38 + , 39 + , 40 + , 41 + , 42 + , 43 + , 44 + , 45 + , 46 + , 47 + , 48 + , 49 + , 50 + , 1.0 + , 2.0 + , 3.0 + , 4.0 + , 5.0 + , 6.0 + , 7.0 + , 8.0 + , 9.0 + , 10.0 + , 11.0 + , 12.0 + , 13.0 + , 14.0 + , 15.0 + , 16.0 + , 17.0 + , 18.0 + , 19.0 + , 20.0 + , 21.0 + , 22.0 + , 23.0 + , 24.0 + , 25.0 + , 26.0 + , 27.0 + , 28.0 + , 29.0 + , 30.0 + , 31.0 + , 32.0 + , 33.0 + , 34.0 + , 35.0 + , 36.0 + , 37.0 + , 38.0 + , 39.0 + , 40.0 + , 41.0 + , 42.0 + , 43.0 + , 44.0 + , 45.0 + , 46.0 + , 47.0 + , 48.0 + , 49.0 + , 50.0 + , 3.14 + , 2.22 + , 8.55 + , 23.3 + , 12.22222235049450945049504950 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" + expected + actual + ) + +parseIntsAndDoublesAsDoublesWithManyExamples :: Test +parseIntsAndDoublesAsDoublesWithManyExamples = + let afterParse :: [Double] + afterParse = + [ 1 + , 2 + , 3 + , 4 + , 5 + , 6 + , 7 + , 8 + , 9 + , 10 + , 11 + , 12 + , 13 + , 14 + , 15 + , 16 + , 17 + , 18 + , 19 + , 20 + , 21 + , 22 + , 23 + , 24 + , 25 + , 26 + , 27 + , 28 + , 29 + , 30 + , 31 + , 32 + , 33 + , 34 + , 35 + , 36 + , 37 + , 38 + , 39 + , 40 + , 41 + , 42 + , 43 + , 44 + , 45 + , 46 + , 47 + , 48 + , 49 + , 50 + , 1.0 + , 2.0 + , 3.0 + , 4.0 + , 5.0 + , 6.0 + , 7.0 + , 8.0 + , 9.0 + , 10.0 + , 11.0 + , 12.0 + , 13.0 + , 14.0 + , 15.0 + , 16.0 + , 17.0 + , 18.0 + , 19.0 + , 20.0 + , 21.0 + , 22.0 + , 23.0 + , 24.0 + , 25.0 + , 26.0 + , 27.0 + , 28.0 + , 29.0 + , 30.0 + , 31.0 + , 32.0 + , 33.0 + , 34.0 + , 35.0 + , 36.0 + , 37.0 + , 38.0 + , 39.0 + , 40.0 + , 41.0 + , 42.0 + , 43.0 + , 44.0 + , 45.0 + , 46.0 + , 47.0 + , 48.0 + , 49.0 + , 50.0 + , 3.14 + , 2.22 + , 8.55 + , 23.3 + , 12.22222235049450945049504950 + ] + beforeParse :: [T.Text] + beforeParse = + [ "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.UnboxedColumn $ VU.fromList afterParse + actual = D.parseDefault 50 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff :: Test +parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff = + let afterParse :: [Maybe Double] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just 1.0 + , Just 2.0 + , Just 3.0 + , Just 4.0 + , Just 5.0 + , Just 6.0 + , Just 7.0 + , Just 8.0 + , Just 9.0 + , Just 10.0 + , Just 11.0 + , Just 12.0 + , Just 13.0 + , Just 14.0 + , Just 15.0 + , Just 16.0 + , Just 17.0 + , Just 18.0 + , Just 19.0 + , Just 20.0 + , Just 21.0 + , Just 22.0 + , Just 23.0 + , Just 24.0 + , Just 25.0 + , Just 26.0 + , Just 27.0 + , Just 28.0 + , Just 29.0 + , Just 30.0 + , Just 31.0 + , Just 32.0 + , Just 33.0 + , Just 34.0 + , Just 35.0 + , Just 36.0 + , Just 37.0 + , Just 38.0 + , Just 39.0 + , Just 40.0 + , Just 41.0 + , Just 42.0 + , Just 43.0 + , Just 44.0 + , Just 45.0 + , Just 46.0 + , Just 47.0 + , Just 48.0 + , Just 49.0 + , Just 50.0 + , Just 1.0 + , Just 2.0 + , Just 3.0 + , Just 4.0 + , Just 5.0 + , Just 6.0 + , Just 7.0 + , Just 8.0 + , Just 9.0 + , Just 10.0 + , Just 11.0 + , Just 12.0 + , Just 13.0 + , Just 14.0 + , Just 15.0 + , Just 16.0 + , Just 17.0 + , Just 18.0 + , Just 19.0 + , Just 20.0 + , Just 21.0 + , Just 22.0 + , Just 23.0 + , Just 24.0 + , Just 25.0 + , Just 26.0 + , Just 27.0 + , Just 28.0 + , Just 29.0 + , Just 30.0 + , Just 31.0 + , Just 32.0 + , Just 33.0 + , Just 34.0 + , Just 35.0 + , Just 36.0 + , Just 37.0 + , Just 38.0 + , Just 39.0 + , Just 40.0 + , Just 41.0 + , Just 42.0 + , Just 43.0 + , Just 44.0 + , Just 45.0 + , Just 46.0 + , Just 47.0 + , Just 48.0 + , Just 49.0 + , Just 50.0 + , Just 3.14 + , Just 2.22 + , Just 8.55 + , Just 23.3 + , Just 12.22222235049451 + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff :: + Test +parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff = + let afterParse :: [Maybe Double] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just 1.0 + , Just 2.0 + , Just 3.0 + , Just 4.0 + , Just 5.0 + , Just 6.0 + , Just 7.0 + , Just 8.0 + , Just 9.0 + , Just 10.0 + , Just 11.0 + , Just 12.0 + , Just 13.0 + , Just 14.0 + , Just 15.0 + , Just 16.0 + , Just 17.0 + , Just 18.0 + , Just 19.0 + , Just 20.0 + , Just 21.0 + , Just 22.0 + , Just 23.0 + , Just 24.0 + , Just 25.0 + , Just 26.0 + , Just 27.0 + , Just 28.0 + , Just 29.0 + , Just 30.0 + , Just 31.0 + , Just 32.0 + , Just 33.0 + , Just 34.0 + , Just 35.0 + , Just 36.0 + , Just 37.0 + , Just 38.0 + , Just 39.0 + , Just 40.0 + , Just 41.0 + , Just 42.0 + , Just 43.0 + , Just 44.0 + , Just 45.0 + , Just 46.0 + , Just 47.0 + , Just 48.0 + , Just 49.0 + , Just 50.0 + , Just 1.0 + , Just 2.0 + , Just 3.0 + , Just 4.0 + , Just 5.0 + , Just 6.0 + , Just 7.0 + , Just 8.0 + , Just 9.0 + , Just 10.0 + , Just 11.0 + , Just 12.0 + , Just 13.0 + , Just 14.0 + , Just 15.0 + , Just 16.0 + , Just 17.0 + , Just 18.0 + , Just 19.0 + , Just 20.0 + , Just 21.0 + , Just 22.0 + , Just 23.0 + , Just 24.0 + , Just 25.0 + , Just 26.0 + , Just 27.0 + , Just 28.0 + , Just 29.0 + , Just 30.0 + , Just 31.0 + , Just 32.0 + , Just 33.0 + , Just 34.0 + , Just 35.0 + , Just 36.0 + , Just 37.0 + , Just 38.0 + , Just 39.0 + , Just 40.0 + , Just 41.0 + , Just 42.0 + , Just 43.0 + , Just 44.0 + , Just 45.0 + , Just 46.0 + , Just 47.0 + , Just 48.0 + , Just 49.0 + , Just 50.0 + , Just 3.14 + , Just 2.22 + , Just 8.55 + , Just 23.3 + , Just 12.22222235049451 + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "21" + , "22" + , "23" + , "24" + , "25" + , "26" + , "27" + , "28" + , "29" + , "30" + , "31" + , "32" + , "33" + , "34" + , "35" + , "36" + , "37" + , "38" + , "39" + , "40" + , "41" + , "42" + , "43" + , "44" + , "45" + , "46" + , "47" + , "48" + , "49" + , "50" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "31.0" + , "32.0" + , "33.0" + , "34.0" + , "35.0" + , "36.0" + , "37.0" + , "38.0" + , "39.0" + , "40.0" + , "41.0" + , "42.0" + , "43.0" + , "44.0" + , "45.0" + , "46.0" + , "47.0" + , "48.0" + , "49.0" + , "50.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff :: + Test +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff = + let afterParse :: [Maybe T.Text] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "1" + , Just "2" + , Just "3" + , Just "4" + , Just "5" + , Just "6" + , Just "7" + , Just "8" + , Just "9" + , Just "10" + , Just "11" + , Just "12" + , Just "13" + , Just "14" + , Just "15" + , Just "16" + , Just "17" + , Just "18" + , Just "19" + , Just "20" + , Just "1.0" + , Just "2.0" + , Just "3.0" + , Just "4.0" + , Just "5.0" + , Just "6.0" + , Just "7.0" + , Just "8.0" + , Just "9.0" + , Just "10.0" + , Just "11.0" + , Just "12.0" + , Just "13.0" + , Just "14.0" + , Just "15.0" + , Just "16.0" + , Just "17.0" + , Just "18.0" + , Just "19.0" + , Just "20.0" + , Just "21.0" + , Just "22.0" + , Just "23.0" + , Just "24.0" + , Just "25.0" + , Just "26.0" + , Just "27.0" + , Just "28.0" + , Just "29.0" + , Just "30.0" + , Just "3.14" + , Just "2.22" + , Just "8.55" + , Just "23.3" + , Just "12.22222235049451" + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with just one example, when safeRead is off" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff :: + Test +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff = + let afterParse :: [Maybe T.Text] + afterParse = + [ Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Nothing + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "NaN" + , Just "N/A" + , Just "1" + , Just "2" + , Just "3" + , Just "4" + , Just "5" + , Just "6" + , Just "7" + , Just "8" + , Just "9" + , Just "10" + , Just "11" + , Just "12" + , Just "13" + , Just "14" + , Just "15" + , Just "16" + , Just "17" + , Just "18" + , Just "19" + , Just "20" + , Just "1.0" + , Just "2.0" + , Just "3.0" + , Just "4.0" + , Just "5.0" + , Just "6.0" + , Just "7.0" + , Just "8.0" + , Just "9.0" + , Just "10.0" + , Just "11.0" + , Just "12.0" + , Just "13.0" + , Just "14.0" + , Just "15.0" + , Just "16.0" + , Just "17.0" + , Just "18.0" + , Just "19.0" + , Just "20.0" + , Just "21.0" + , Just "22.0" + , Just "23.0" + , Just "24.0" + , Just "25.0" + , Just "26.0" + , Just "27.0" + , Just "28.0" + , Just "29.0" + , Just "30.0" + , Just "3.14" + , Just "2.22" + , Just "8.55" + , Just "23.3" + , Just "12.22222235049451" + ] + beforeParse :: [T.Text] + beforeParse = + [ "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "NaN" + , "N/A" + , "1" + , "2" + , "3" + , "4" + , "5" + , "6" + , "7" + , "8" + , "9" + , "10" + , "11" + , "12" + , "13" + , "14" + , "15" + , "16" + , "17" + , "18" + , "19" + , "20" + , "1.0" + , "2.0" + , "3.0" + , "4.0" + , "5.0" + , "6.0" + , "7.0" + , "8.0" + , "9.0" + , "10.0" + , "11.0" + , "12.0" + , "13.0" + , "14.0" + , "15.0" + , "16.0" + , "17.0" + , "18.0" + , "19.0" + , "20.0" + , "21.0" + , "22.0" + , "23.0" + , "24.0" + , "25.0" + , "26.0" + , "27.0" + , "28.0" + , "29.0" + , "30.0" + , "3.14" + , "2.22" + , "8.55" + , "23.3" + , "12.22222235049451" + ] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with many examples, when safeRead is off" + expected + actual + ) + +-- 5. EDGE CASES THAT HAVE TO BE INTERPRETED CORRECTLY + +parseManyNullishAndOneInt :: Test +parseManyNullishAndOneInt = + let afterParse :: [Maybe Int] + afterParse = replicate 100 Nothing ++ [Just 100000] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["100000"] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseManyNullishAndOneDouble :: Test +parseManyNullishAndOneDouble = + let afterParse :: [Maybe Double] + afterParse = replicate 100 Nothing ++ [Just 3.14] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["3.14"] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseManyNullishAndOneDate :: Test +parseManyNullishAndOneDate = + let afterParse :: [Maybe Day] + afterParse = replicate 100 Nothing ++ [Just $ fromGregorian 2024 12 25] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["2024-12-25"] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseManyNullishAndIncorrectDates :: Test +parseManyNullishAndIncorrectDates = + let afterParse :: [Maybe T.Text] + afterParse = replicate 100 Nothing ++ [Just "2024-12-25", Just "2024-12-w6"] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["2024-12-25", "2024-12-w6"] + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseRepeatedNullish :: Test +parseRepeatedNullish = + let afterParse :: [Maybe T.Text] + afterParse = replicate 100 Nothing + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" + expected = DI.OptionalColumn $ V.fromList afterParse + actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseTests :: [Test] +parseTests = + [ -- 1. SIMPLE CASES + TestLabel "parseInts" parseInts + , TestLabel "parseDoubles" parseDoubles + , TestLabel "parseDates" parseDates + , TestLabel "parseTexts" parseTexts + , -- 2. COMBINATION CASES + TestLabel "parseIntsAndDoublesAsDoubles" parseIntsAndDoublesAsDoubles + , TestLabel "parseIntsAndDatesAsTexts" parseIntsAndDatesAsTexts + , TestLabel "parseTextsAndDoublesAsTexts" parseTextsAndDoublesAsTexts + , TestLabel "parseDatesAndTextsAsTexts" parseDatesAndTextsAsTexts + , -- 3A. PARSING WITH SAFEREAD OFF + TestLabel "parseIntsWithoutSafeRead" parseIntsWithoutSafeRead + , TestLabel "parseDoublesWithoutSafeRead" parseDoublesWithoutSafeRead + , TestLabel "parseDatesWithoutSafeRead" parseDatesWithoutSafeRead + , TestLabel "parseTextsWithoutSafeRead" parseTextsWithoutSafeRead + , TestLabel + "parseIntsAndEmptyStringsWithoutSafeRead" + parseIntsAndEmptyStringsWithoutSafeRead + , TestLabel + "parseIntsAndDoublesAndEmptyStringsWithoutSafeRead" + parseIntsAndDoublesAndEmptyStringsWithoutSafeRead + , TestLabel + "parseDatesAndEmptyStringsWithoutSafeRead" + parseDatesAndEmptyStringsWithoutSafeRead + , TestLabel + "parseTextsAndEmptyStringsWithoutSafeRead" + parseTextsAndEmptyStringsWithoutSafeRead + , TestLabel + "parseIntsAndNullishStringsWithoutSafeRead" + parseIntsAndNullishStringsWithoutSafeRead + , TestLabel + "parseIntsAndDoublesAndNullishStringsWithoutSafeRead" + parseIntsAndDoublesAndNullishStringsWithoutSafeRead + , TestLabel + "parseIntsAndNullishAndEmptyStringsWithoutSafeRead" + parseIntsAndNullishAndEmptyStringsWithoutSafeRead + , TestLabel + "parseTextsAndEmptyAndNullishStringsWithoutSafeRead" + parseTextsAndEmptyAndNullishStringsWithoutSafeRead + , -- 3B. PARSING WITH SAFEREAD ON + TestLabel + "parseIntsAndEmptyStringsWithSafeRead" + parseIntsAndEmptyStringsWithSafeRead + , TestLabel + "parseIntsAndDoublesAndEmptyStringsWithSafeRead" + parseIntsAndDoublesAndEmptyStringsWithSafeRead + , TestLabel + "parseDatesAndEmptyStringsWithSafeRead" + parseDatesAndEmptyStringsWithSafeRead + , TestLabel + "parseTextsAndEmptyStringsWithSafeRead" + parseTextsAndEmptyStringsWithSafeRead + , TestLabel + "parseIntsAndNullishStringsWithSafeRead" + parseIntsAndNullishStringsWithSafeRead + , TestLabel + "parseIntsAndDoublesAndNullishStringsWithSafeRead" + parseIntsAndDoublesAndNullishStringsWithSafeRead + , TestLabel + "parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead" + parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead + , TestLabel + "parseIntsAndNullishAndEmptyStringsWithSafeRead" + parseIntsAndNullishAndEmptyStringsWithSafeRead + , TestLabel + "parseTextsAndEmptyAndNullishStringsWithSafeRead" + parseTextsAndEmptyAndNullishStringsWithSafeRead + , -- 4A. PARSING MUST NOT DEPEND ON THE NUMBER OF EXAMPLES + TestLabel "parseIntsWithOneExample" parseIntsWithOneExample + , TestLabel "parseIntsWithTwentyFiveExamples" parseIntsWithTwentyFiveExamples + , TestLabel "parseIntsWithFortyNineExamples" parseIntsWithFortyNineExamples + , TestLabel "parseDatesWithOneExample" parseDatesWithOneExample + , TestLabel "parseDatesWithFifteenExamples" parseDatesWithFifteenExamples + , TestLabel + "parseIntsAndDoublesAsDoublesWithOneExample" + parseIntsAndDoublesAsDoublesWithOneExample + , TestLabel + "parseIntsAndDoublesAsDoublesWithManyExamples" + parseIntsAndDoublesAsDoublesWithManyExamples + , TestLabel + "parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff" + parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff + , TestLabel + "parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff" + parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff + , TestLabel + "parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff" + parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff + , TestLabel + "parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff" + parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff + , -- 5. EDGE CASES THAT HAVE TO BE PARSED CORRECTLY + TestLabel "parseManyNullishAndOneInt" parseManyNullishAndOneInt + , TestLabel "parseManyNullishAndOneDouble" parseManyNullishAndOneDouble + , TestLabel "parseRepeatedNullish" parseRepeatedNullish + ] + +tests :: Test +tests = + TestList $ + dimensionsTest + ++ Operations.Aggregations.tests + ++ Operations.Apply.tests + ++ Operations.Core.tests + ++ Operations.Derive.tests + ++ Operations.Filter.tests + ++ Operations.GroupBy.tests + ++ Operations.InsertColumn.tests + ++ Operations.Join.tests ++ Operations.Merge.tests ++ Operations.ReadCsv.tests ++ Operations.Sort.tests diff --git a/tests/Operations/Filter.hs b/tests/Operations/Filter.hs --- a/tests/Operations/Filter.hs +++ b/tests/Operations/Filter.hs @@ -5,6 +5,7 @@ import qualified Data.Text as T import qualified DataFrame as D +import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI import Assertions @@ -32,7 +33,7 @@ ( assertExpectException "[Error Case]" (D.columnNotFound "test0" "filter" (D.columnNames testData)) - (print $ D.filter @Int "test0" even testData) + (print $ D.filter (F.col @Int "test0") even testData) ) filterColumnWrongType :: Test @@ -41,7 +42,7 @@ ( assertExpectException "[Error Case]" (D.typeMismatchError (show $ typeRep @Integer) (show $ typeRep @Int)) - (print $ D.filter @Integer "test1" even testData) + (print $ D.filter (F.col @Integer "test1") even testData) ) filterByColumnDoesNotExist :: Test @@ -50,7 +51,7 @@ ( assertExpectException "[Error Case]" (D.columnNotFound "test0" "filter" (D.columnNames testData)) - (print $ D.filterBy @Int even "test0" testData) + (print $ D.filterBy even (F.col @Int "test0") testData) ) filterByColumnWrongType :: Test @@ -59,7 +60,7 @@ ( assertExpectException "[Error Case]" (D.typeMismatchError (show $ typeRep @Integer) (show $ typeRep @Int)) - (print $ D.filterBy @Integer even "test1" testData) + (print $ D.filterBy even (F.col @Integer "test1") testData) ) filterColumnInexistentValues :: Test @@ -68,7 +69,7 @@ ( assertEqual "Non existent filter value returns no rows" (0, 8) - (D.dimensions $ D.filter @Int "test1" (< 0) testData) + (D.dimensions $ D.filter (F.col @Int "test1") (< 0) testData) ) filterColumnAllValues :: Test @@ -77,7 +78,7 @@ ( assertEqual "Filters all columns" (26, 8) - (D.dimensions $ D.filter @Int "test1" (const True) testData) + (D.dimensions $ D.filter (F.col @Int "test1") (const True) testData) ) filterJustWAI :: Test diff --git a/tests/Operations/Join.hs b/tests/Operations/Join.hs new file mode 100644 --- /dev/null +++ b/tests/Operations/Join.hs @@ -0,0 +1,85 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Operations.Join where + +import Data.Text (Text) +import qualified DataFrame as D +import DataFrame.Operations.Join +import Test.HUnit + +df1 :: D.DataFrame +df1 = + D.fromNamedColumns + [ ("key", D.fromList ["K0" :: Text, "K1", "K2", "K3", "K4", "K5"]) + , ("A", D.fromList ["A0" :: Text, "A1", "A2", "A3", "A4", "A5"]) + ] + +df2 :: D.DataFrame +df2 = + D.fromNamedColumns + [ ("key", D.fromList ["K0" :: Text, "K1", "K2"]) + , ("B", D.fromList ["B0" :: Text, "B1", "B2"]) + ] + +testInnerJoin :: Test +testInnerJoin = + TestCase + ( assertEqual + "Test inner join with single key" + ( D.fromNamedColumns + [ ("key", D.fromList ["K0" :: Text, "K1", "K2"]) + , ("A", D.fromList ["A0" :: Text, "A1", "A2"]) + , ("B", D.fromList ["B0" :: Text, "B1", "B2"]) + ] + ) + (D.sortBy D.Ascending ["key"] (innerJoin ["key"] df1 df2)) + ) + +testLeftJoin :: Test +testLeftJoin = + TestCase + ( assertEqual + "Test left join with single key" + ( D.fromNamedColumns + [ ("key", D.fromList ["K0" :: Text, "K1", "K2", "K3", "K4", "K5"]) + , ("A", D.fromList ["A0" :: Text, "A1", "A2", "A3", "A4", "A5"]) + , ("B", D.fromList [Just "B0", Just "B1" :: Maybe Text, Just "B2"]) + ] + ) + (D.sortBy D.Ascending ["key"] (leftJoin ["key"] df2 df1)) + ) + +testRightJoin :: Test +testRightJoin = + TestCase + ( assertEqual + "Test right join with single key" + ( D.fromNamedColumns + [ ("key", D.fromList ["K0" :: Text, "K1", "K2"]) + , ("A", D.fromList ["A0" :: Text, "A1", "A2"]) + , ("B", D.fromList ["B0" :: Text, "B1", "B2"]) + ] + ) + (D.sortBy D.Ascending ["key"] (rightJoin ["key"] df2 df1)) + ) + +testFullOuterJoin :: Test +testFullOuterJoin = + TestCase + ( assertEqual + "Test full outer join with single key" + ( D.fromNamedColumns + [ ("key", D.fromList ["K0" :: Text, "K1", "K2", "K3", "K4", "K5"]) + , ("A", D.fromList ["A0" :: Text, "A1", "A2", "A3", "A4", "A5"]) + , ("B", D.fromList [Just "B0", Just "B1" :: Maybe Text, Just "B2"]) + ] + ) + (D.sortBy D.Ascending ["key"] (fullOuterJoin ["key"] df2 df1)) + ) + +tests :: [Test] +tests = + [ TestLabel "innerJoin" testInnerJoin + , TestLabel "leftJoin" testLeftJoin + , TestLabel "rightJoin" testRightJoin + ] diff --git a/tests/Operations/Statistics.hs b/tests/Operations/Statistics.hs --- a/tests/Operations/Statistics.hs +++ b/tests/Operations/Statistics.hs @@ -43,7 +43,7 @@ TestCase ( assertBool "Skewness of a data set with the same elements" - (isNaN (D.skewness' (VU.fromList $ replicate 10 42.0))) + (isNaN (D.skewness' @Double (VU.fromList $ replicate 10 42.0))) ) skewnessOfSymmetricDataSet :: Test @@ -51,7 +51,7 @@ TestCase ( assertEqual "Skewness of a symmetric data set" - (D.skewness' (VU.fromList [-3.0, -2.0, -1.5, 0, 1.5, 2.0, 3.0])) + (D.skewness' (VU.fromList [-3.0 :: Double, -2.0, -1.5, 0, 1.5, 2.0, 3.0])) 0 ) @@ -60,7 +60,10 @@ TestCase ( assertBool "Skewness of a simple data set" - ( abs (D.skewness' (VU.fromList [25, 28, 26, 30, 40, 50, 40]) - 0.566_731_633_676) + ( abs + ( D.skewness' (VU.fromList [25 :: Int, 28, 26, 30, 40, 50, 40]) + - 0.566_731_633_676 + ) < 1e-12 ) ) @@ -70,7 +73,7 @@ TestCase ( assertEqual "Skewness of an empty data set" - (D.skewness' (VU.fromList [])) + (D.skewness' @Double (VU.fromList [])) 0 ) @@ -82,7 +85,7 @@ ( D.quantiles' (VU.fromList [0, 1, 2]) 2 - (VU.fromList [179.94, 231.94, 839.06, 534.23, 248.94]) + (VU.fromList [179.94 :: Double, 231.94, 839.06, 534.23, 248.94]) ) (VU.fromList [179.94, 248.94, 839.06]) ) @@ -95,7 +98,7 @@ ( D.quantiles' (VU.fromList [0, 1, 2]) 2 - (VU.fromList [179.94, 231.94, 839.06, 534.23, 248.94, 276.37]) + (VU.fromList [179.94 :: Double, 231.94, 839.06, 534.23, 248.94, 276.37]) ) (VU.fromList [179.94, 262.655, 839.06]) ) @@ -108,7 +111,7 @@ ( D.quantiles' (VU.fromList [0, 1, 2, 3, 4]) 4 - (VU.fromList [3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20]) + (VU.fromList [3 :: Int, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20]) ) (VU.fromList [3, 7.5, 9, 14, 20]) ) @@ -121,7 +124,7 @@ ( D.quantiles' (VU.fromList [0, 1, 2, 3, 4]) 4 - (VU.fromList [3, 6, 7, 8, 8, 10, 13, 15, 16, 20]) + (VU.fromList [3 :: Int, 6, 7, 8, 8, 10, 13, 15, 16, 20]) ) (VU.fromList [3, 7.25, 9, 14.5, 20]) ) @@ -134,7 +137,7 @@ ( D.quantiles' (VU.fromList [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) 10 - (VU.fromList [4, 7, 3, 1, 11, 6, 2, 9, 8, 10, 5]) + (VU.fromList [4 :: Int, 7, 3, 1, 11, 6, 2, 9, 8, 10, 5]) ) (VU.fromList [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) ) @@ -144,7 +147,9 @@ TestCase ( assertEqual "Inter quartile range of an odd length data set" - (D.interQuartileRange' (VU.fromList [3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20])) + ( D.interQuartileRange' + (VU.fromList [3 :: Int, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20]) + ) 6.5 ) @@ -153,7 +158,7 @@ TestCase ( assertEqual "Inter quartile range of an even length data set" - (D.interQuartileRange' (VU.fromList [3, 6, 7, 8, 8, 10, 13, 15, 16, 20])) + (D.interQuartileRange' (VU.fromList [3 :: Int, 6, 7, 8, 8, 10, 13, 15, 16, 20])) 7.25 ) @@ -163,7 +168,7 @@ ( assertExpectException "[Error Case]" (D.wrongQuantileNumberError 1) - (print $ D.quantiles' (VU.fromList [0]) 1 (VU.fromList [1, 2, 3, 4, 5])) + (print $ D.quantiles' (VU.fromList [0]) 1 (VU.fromList [1 :: Int, 2, 3, 4, 5])) ) wrongQuantileIndex :: Test @@ -172,7 +177,7 @@ ( assertExpectException "[Error Case]" (D.wrongQuantileIndexError (VU.fromList [5]) 4) - (print $ D.quantiles' (VU.fromList [5]) 4 (VU.fromList [1, 2, 3, 4, 5])) + (print $ D.quantiles' (VU.fromList [5]) 4 (VU.fromList [1 :: Int, 2, 3, 4, 5])) ) tests :: [Test] diff --git a/tests/Parquet.hs b/tests/Parquet.hs --- a/tests/Parquet.hs +++ b/tests/Parquet.hs @@ -1,8 +1,10 @@ {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} module Parquet where import qualified DataFrame as D +import qualified DataFrame.Functions as F import Data.Int import Data.Text (Text) @@ -64,7 +66,7 @@ TestCase ( assertEqual "allTypesPlainSnappy" - (D.filter "id" (`elem` [6 :: Int32, 7]) allTypes) + (D.filter (F.col @Int32 "id") (`elem` [6, 7]) allTypes) (unsafePerformIO (D.readParquet "./tests/data/alltypes_plain.snappy.parquet")) ) @@ -73,7 +75,7 @@ TestCase ( assertEqual "allTypesPlainSnappy" - (D.filter "id" (`elem` [0 :: Int32, 1]) allTypes) + (D.filter (F.col @Int32 "id") (`elem` [0, 1]) allTypes) (unsafePerformIO (D.readParquet "./tests/data/alltypes_dictionary.parquet")) )