diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Revision history for dataframe +## 0.4.1.0 +* Improve signal handling of dataframe repl. +* `writeCsv` not correctly writes `Maybe` values (thanks to @mcoady). +* Create a boilerplate package in cache that will be used to start repl. +* Tree implementation is now a TAO tree instead of greedy cart trees. +* Add `sampleM` and `takeM` functions. + ## 0.4.0.10 * License in cabal was wrong. * Remove ollama-haskell dependencies. diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -1,32 +1,51 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE ScopedTypeVariables #-} + module Main where +import Control.Exception (bracket) import Control.Monad import Data.Time (NominalDiffTime, diffUTCTime, getCurrentTime) import System.Directory ( XdgDirectory (..), createDirectoryIfMissing, doesFileExist, + getCurrentDirectory, getModificationTime, getXdgDirectory, ) +import System.Exit (ExitCode (..)) import System.FilePath (()) import System.Process +#ifndef mingw32_HOST_OS +import System.Posix.Signals (Handler (..), installHandler, sigINT) +#endif + main :: IO () main = do cacheDir <- getXdgDirectory XdgCache "dataframe_repl" + currDir <- getCurrentDirectory + print currDir createDirectoryIfMissing True cacheDir - let filepath = cacheDir "dataframe.ghci" - shouldDownload <- needsUpdate filepath + let ghciScript = cacheDir "dataframe.ghci" + projectFile = cacheDir "cabal.project" + cabalFile = cacheDir "df-repl.cabal" + mainFile = cacheDir "Main.hs" + -- Ensure the directory is a valid Cabal project with a local target to repl into. + ensureBoilerplate projectFile cabalFile mainFile + + -- Keep your ghci script updated (weekly). + shouldDownload <- needsUpdate ghciScript when shouldDownload $ do putStrLn "\ESC[92mDownloading latest version of dataframe config...\ESC[0m" output <- readProcess "curl" [ "--output" - , filepath + , ghciScript , "https://raw.githubusercontent.com/mchav/dataframe/refs/heads/main/dataframe.ghci" ] "" @@ -37,15 +56,82 @@ let command = "cabal" args = [ "repl" + , "exe:df-repl" + , "--project-file=" ++ projectFile , "-O2" - , "--build-depends" - , "dataframe" - , "--repl-option=-ghci-script=" ++ filepath + , "--repl-option=-ghci-script=" ++ ghciScript ] - (_, _, _, processHandle) <- createProcess (proc command args) - exitCode <- waitForProcess processHandle - pure () + baseCp = + (proc command args) + { cwd = Just currDir + , std_in = Inherit + , std_out = Inherit + , std_err = Inherit + } + +#ifdef mingw32_HOST_OS + cp = baseCp { delegate_ctlc = True } +#else + cp = baseCp +#endif + +#ifndef mingw32_HOST_OS + -- Unix: ignore Ctrl-C in the wrapper so the child handles it. + bracket + (installHandler sigINT Ignore Nothing) + (\old -> installHandler sigINT old Nothing) + (\_ -> runChild cp) +#else + -- Windows: delegate Ctrl-C handling to the child. + runChild cp +#endif + +ensureBoilerplate :: FilePath -> FilePath -> FilePath -> IO () +ensureBoilerplate projectFile cabalFile mainFile = do + -- cabal.project + writeIfMissing + projectFile + $ unlines + [ "packages: ." + ] + + writeIfMissing + cabalFile + $ unlines + [ "cabal-version: 3.0" + , "name: df-repl" + , "version: 0.1.0.0" + , "build-type: Simple" + , "" + , "executable df-repl" + , " main-is: Main.hs" + , " hs-source-dirs: ." + , " default-language: Haskell2010" + , " build-depends: base, dataframe, text, time, random" + ] + + writeIfMissing + mainFile + $ unlines + [ "module Main where" + , "" + , "main :: IO ()" + , "main = putStrLn \"df-repl\"" + ] + +writeIfMissing :: FilePath -> String -> IO () +writeIfMissing fp contents = do + exists <- doesFileExist fp + unless exists $ writeFile fp contents + +runChild :: CreateProcess -> IO () +runChild cp = do + (_, _, _, ph) <- createProcess cp + ec <- waitForProcess ph + case ec of + ExitSuccess -> pure () + ExitFailure n -> fail ("cabal repl failed with exit code " <> show n) oneWeek :: NominalDiffTime oneWeek = 7 * 24 * 60 * 60 diff --git a/app/Synthesis.hs b/app/Synthesis.hs --- a/app/Synthesis.hs +++ b/app/Synthesis.hs @@ -43,7 +43,7 @@ (D.nRows train) combined |> D.filterJust (F.name survived) - |> D.randomSplit (mkStdGen 4232) 0.8 + |> D.randomSplit (mkStdGen 4232) 0.7 -- Split the test out again. test' = D.drop @@ -54,14 +54,18 @@ fitDecisionTree ( defaultTreeConfig { maxTreeDepth = 5 - , minSamplesSplit = 25 - , minLeafSize = 15 + , minSamplesSplit = 10 + , minLeafSize = 3 + , taoIterations = 100 , synthConfig = defaultSynthConfig - { complexityPenalty = 0 + { complexityPenalty = 0.00 , maxExprDepth = 2 , disallowedCombinations = - [(F.name age, F.name fare)] + [ (F.name age, F.name fare) + , ("passenger_class", "number_of_siblings_and_spouses") + , ("passenger_class", "number_of_parents_and_children") + ] } } ) diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 0.4.0.10 +version: 0.4.1.0 synopsis: A fast, safe, and intuitive DataFrame library. @@ -157,7 +157,8 @@ directory >= 1.3.0.0 && < 2, filepath >= 1.4 && < 2, process >= 1.6 && < 2, - time >= 1.12 && < 2 + time >= 1.12 && < 2, + unix >= 2 && < 3 hs-source-dirs: app default-language: Haskell2010 ghc-options: -rtsopts -threaded -with-rtsopts=-N @@ -183,6 +184,7 @@ main-is: Main.hs other-modules: Assertions, Functions, + GenDataFrame, Operations.Aggregations, Operations.Apply, Operations.Core, @@ -194,6 +196,7 @@ Operations.Merge, Operations.ReadCsv, Operations.Sort, + Operations.Subset, Operations.Statistics, Operations.Take, Parquet @@ -201,6 +204,7 @@ dataframe ^>= 0.4, directory >= 1.3.0.0 && < 2, HUnit ^>= 1.6, + QuickCheck >= 2 && < 3, random >= 1 && < 2, random-shuffle >= 0.0.4 && < 1, random >= 1 && < 2, diff --git a/src/DataFrame/DecisionTree.hs b/src/DataFrame/DecisionTree.hs --- a/src/DataFrame/DecisionTree.hs +++ b/src/DataFrame/DecisionTree.hs @@ -1,11 +1,7 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE BangPatterns #-} -{-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-} -{-# LANGUAGE MultiParamTypeClasses #-} -{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -14,19 +10,18 @@ import qualified DataFrame.Functions as F import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn) -import DataFrame.Internal.Expression (Expr (..), eSize, getColumns, prettyPrint) +import DataFrame.Internal.Expression (Expr (..), eSize, getColumns) import DataFrame.Internal.Interpreter (interpret) import DataFrame.Internal.Statistics (percentile', percentileOrd') import DataFrame.Internal.Types import DataFrame.Operations.Core (columnNames, nRows) -import DataFrame.Operations.Statistics (percentile) import DataFrame.Operations.Subset (exclude, filterWhere) -import Control.Exception (SomeException, throw, try) +import Control.Exception (throw) import Control.Monad (guard) import Data.Containers.ListUtils (nubOrd) import Data.Function (on) -import Data.List (foldl', maximumBy, sort, sortBy) +import Data.List (foldl', maximumBy, minimumBy, sort, sortBy) import qualified Data.Map.Strict as M import Data.Maybe import qualified Data.Text as T @@ -35,19 +30,17 @@ import qualified Data.Vector.Unboxed as VU import Type.Reflection (typeRep) -import Data.Text.Read import DataFrame.Functions ((./=), (.<), (.<=), (.==), (.>), (.>=)) -import Debug.Trace (trace) -import GHC.IO.Unsafe (unsafePerformIO) -data TreeConfig - = TreeConfig +data TreeConfig = TreeConfig { maxTreeDepth :: Int , minSamplesSplit :: Int , minLeafSize :: Int , percentiles :: [Int] , expressionPairs :: Int , synthConfig :: SynthConfig + , taoIterations :: Int + , taoConvergenceTol :: Double } deriving (Eq, Show) @@ -83,8 +76,25 @@ , percentiles = [0, 10 .. 100] , expressionPairs = 10 , synthConfig = defaultSynthConfig + , taoIterations = 10 + , taoConvergenceTol = 1e-6 } +data Tree a + = Leaf !a + | Branch !(Expr Bool) !(Tree a) !(Tree a) + deriving (Eq, Show) + +treeDepth :: Tree a -> Int +treeDepth (Leaf _) = 0 +treeDepth (Branch _ l r) = 1 + max (treeDepth l) (treeDepth r) + +treeToExpr :: (Columnable a) => Tree a -> Expr a +treeToExpr (Leaf v) = Lit v +treeToExpr (Branch cond left right) = + F.ifThenElse cond (treeToExpr left) (treeToExpr right) + +-- | Fit a TAO decision tree fitDecisionTree :: forall a. (Columnable a) => @@ -93,70 +103,455 @@ DataFrame -> Expr a fitDecisionTree cfg (Col target) df = - buildTree @a - cfg - (maxTreeDepth cfg) - target - ( nubOrd $ - numericConditions cfg (exclude [target] df) - ++ generateConditionsOld cfg (exclude [target] df) - ) - df + let + conds = + nubOrd $ + numericConditions cfg (exclude [target] df) + ++ generateConditionsOld cfg (exclude [target] df) + + initialTree = buildGreedyTree @a cfg (maxTreeDepth cfg) target conds df + + indices = V.enumFromN 0 (nRows df) + + optimizedTree = taoOptimize @a cfg target conds df indices initialTree + in + pruneExpr (treeToExpr optimizedTree) fitDecisionTree _ expr _ = error $ "Cannot create tree for compound expression: " ++ show expr -buildTree :: +taoOptimize :: forall a. (Columnable a) => TreeConfig -> + T.Text -> -- Target column name + [Expr Bool] -> -- Candidate conditions + DataFrame -> -- Full dataset + V.Vector Int -> -- Indices of points reaching the root + Tree a -> -- Current tree + Tree a +taoOptimize cfg target conds df rootIndices initialTree = + go 0 initialTree (computeTreeLoss @a target df rootIndices initialTree) + where + go :: Int -> Tree a -> Double -> Tree a + go iter tree prevLoss + | iter >= taoIterations cfg = pruneDead tree + | otherwise = + let + tree' = taoIteration @a cfg target conds df rootIndices tree + + newLoss = computeTreeLoss @a target df rootIndices tree' + improvement = prevLoss - newLoss + in + if improvement < taoConvergenceTol cfg + then pruneDead tree' + else go (iter + 1) tree' newLoss + +taoIteration :: + forall a. + (Columnable a) => + TreeConfig -> + T.Text -> + [Expr Bool] -> + DataFrame -> + V.Vector Int -> + Tree a -> + Tree a +taoIteration cfg target conds df rootIndices tree = + let depth = treeDepth tree + in foldl' + (optimizeDepthLevel @a cfg target conds df rootIndices) + tree + [depth, depth - 1 .. 0] -- Bottom to top + +optimizeDepthLevel :: + forall a. + (Columnable a) => + TreeConfig -> + T.Text -> + [Expr Bool] -> + DataFrame -> + V.Vector Int -> + Tree a -> + Int -> -- Target depth + Tree a +optimizeDepthLevel cfg target conds df rootIndices tree = optimizeAtDepth @a cfg target conds df rootIndices tree 0 + +optimizeAtDepth :: + forall a. + (Columnable a) => + TreeConfig -> + T.Text -> + [Expr Bool] -> + DataFrame -> + V.Vector Int -> + Tree a -> Int -> + Int -> + Tree a +optimizeAtDepth cfg target conds df indices tree currentDepth targetDepth + | currentDepth == targetDepth = + optimizeNode @a cfg target conds df indices tree + | otherwise = case tree of + Leaf v -> Leaf v + Branch cond left right -> + let + (indicesL, indicesR) = partitionIndices cond df indices + left' = + optimizeAtDepth @a + cfg + target + conds + df + indicesL + left + (currentDepth + 1) + targetDepth + right' = + optimizeAtDepth @a + cfg + target + conds + df + indicesR + right + (currentDepth + 1) + targetDepth + in + Branch cond left' right' + +optimizeNode :: + forall a. + (Columnable a) => + TreeConfig -> T.Text -> [Expr Bool] -> DataFrame -> - Expr a -buildTree cfg depth target conds df - | depth <= 0 || nRows df <= minSamplesSplit cfg = - Lit (majorityValue @a target df) + V.Vector Int -> + Tree a -> + Tree a +optimizeNode cfg target conds df indices tree + | V.null indices = tree + | otherwise = case tree of + Leaf _ -> Leaf (majorityValueFromIndices @a target df indices) + Branch oldCond left right -> + let + newCond = findBestSplitTAO @a cfg target conds df indices left right oldCond + + (newIndicesL, newIndicesR) = partitionIndices newCond df indices + in + if V.length newIndicesL < minLeafSize cfg + || V.length newIndicesR < minLeafSize cfg + then Leaf (majorityValueFromIndices @a target df indices) + else Branch newCond left right + +findBestSplitTAO :: + forall a. + (Columnable a) => + TreeConfig -> + T.Text -> + [Expr Bool] -> + DataFrame -> + V.Vector Int -> + Tree a -> -- Left subtree (FIXED) + Tree a -> -- Right subtree (FIXED) + Expr Bool -> -- Current condition (fallback) + Expr Bool +findBestSplitTAO cfg target conds df indices leftTree rightTree currentCond + | V.null indices = currentCond + | null validConds = currentCond | otherwise = - case findBestSplit @a cfg target conds df of - Nothing -> Lit (majorityValue @a target df) - Just bestCond -> - let (dfTrue, dfFalse) = partitionDataFrame bestCond df - in if nRows dfTrue == 0 || nRows dfFalse == 0 - then Lit (majorityValue @a target df) - else - pruneTree - ( F.ifThenElse - bestCond - (buildTree @a cfg (depth - 1) target conds dfTrue) - (buildTree @a cfg (depth - 1) target conds dfFalse) - ) + let + carePoints = identifyCarePoints @a target df indices leftTree rightTree + in + if null carePoints + then currentCond + else + let + evalSplit :: Expr Bool -> Int + evalSplit cond = countCarePointErrors cond df carePoints -pruneTree :: forall a. (Columnable a, Eq a) => Expr a -> Expr a -pruneTree (If cond trueBranch falseBranch) = + evalWithPenalty c = + let errors = evalSplit c + penalty = + floor + ( complexityPenalty (synthConfig cfg) + * fromIntegral (eSize c) + ) + in errors + penalty + + sortedConds = + take (expressionPairs cfg) $ + sortBy (compare `on` evalWithPenalty) validConds + + expandedConds = + boolExprs + df + sortedConds + sortedConds + 0 + (boolExpansion (synthConfig cfg)) + in + if null expandedConds + then currentCond + else minimumBy (compare `on` evalWithPenalty) expandedConds + where + validConds = filter isValidSplit conds + isValidSplit c = + let (t, f) = partitionIndices c df indices + in V.length t >= minLeafSize cfg && V.length f >= minLeafSize cfg + +-- | A care point with its index and which direction leads to correct classification +data CarePoint = CarePoint + { cpIndex :: !Int + , cpCorrectDir :: !Direction -- Which child classifies this point correctly + } + deriving (Eq, Show) + +data Direction = GoLeft | GoRight + deriving (Eq, Show) + +{- | Identify care points: points where exactly one subtree classifies correctly + + For each point reaching the node: + 1. Compute what label the left subtree would predict + 2. Compute what label the right subtree would predict + 3. If exactly one matches the true label, it's a care point + 4. Record which direction leads to correct classification +-} +identifyCarePoints :: + forall a. + (Columnable a) => + T.Text -> + DataFrame -> + V.Vector Int -> + Tree a -> -- Left subtree + Tree a -> -- Right subtree + [CarePoint] +identifyCarePoints target df indices leftTree rightTree = + case interpret @a df (Col target) of + Left _ -> [] + Right (TColumn col) -> + case toVector @a col of + Left _ -> [] + Right targetVals -> + V.toList $ V.mapMaybe (checkPoint targetVals) indices + where + checkPoint :: V.Vector a -> Int -> Maybe CarePoint + checkPoint targetVals idx = + let + trueLabel = targetVals V.! idx + leftPred = predictWithTree @a target df idx leftTree + rightPred = predictWithTree @a target df idx rightTree + leftCorrect = leftPred == trueLabel + rightCorrect = rightPred == trueLabel + in + case (leftCorrect, rightCorrect) of + (True, False) -> Just $ CarePoint idx GoLeft + (False, True) -> Just $ CarePoint idx GoRight + _ -> Nothing -- Don't-care point (both correct or both wrong) + +-- | Predict the label for a single point using a fixed tree +predictWithTree :: + forall a. + (Columnable a) => + T.Text -> + DataFrame -> + Int -> -- Row index + Tree a -> + a +predictWithTree target df idx (Leaf v) = v +predictWithTree target df idx (Branch cond left right) = + case interpret @Bool df cond of + Left _ -> predictWithTree @a target df idx left -- Default to left on error + Right (TColumn col) -> + case toVector @Bool col of + Left _ -> predictWithTree @a target df idx left + Right boolVals -> + if boolVals V.! idx + then predictWithTree @a target df idx left + else predictWithTree @a target df idx right + +countCarePointErrors :: Expr Bool -> DataFrame -> [CarePoint] -> Int +countCarePointErrors cond df carePoints = + case interpret @Bool df cond of + Left _ -> length carePoints + Right (TColumn col) -> + case toVector @Bool col of + Left _ -> length carePoints + Right boolVals -> + length $ filter (isMisclassified boolVals) carePoints + where + isMisclassified :: V.Vector Bool -> CarePoint -> Bool + isMisclassified boolVals cp = + let goesLeft = boolVals V.! cpIndex cp + shouldGoLeft = cpCorrectDir cp == GoLeft + in goesLeft /= shouldGoLeft + +partitionIndices :: + Expr Bool -> DataFrame -> V.Vector Int -> (V.Vector Int, V.Vector Int) +partitionIndices cond df indices = + case interpret @Bool df cond of + Left _ -> (indices, V.empty) + Right (TColumn col) -> + case toVector @Bool col of + Left _ -> (indices, V.empty) + Right boolVals -> + V.partition (boolVals V.!) indices + +majorityValueFromIndices :: + forall a. + (Columnable a) => + T.Text -> + DataFrame -> + V.Vector Int -> + a +majorityValueFromIndices target df indices = + case interpret @a df (Col target) of + Left e -> throw e + Right (TColumn col) -> + case toVector @a col of + Left e -> throw e + Right vals -> + let counts = + V.foldl' + (\acc i -> M.insertWith (+) (vals V.! i) 1 acc) + M.empty + indices + in if M.null counts + then error "Empty indices in majorityValueFromIndices" + else fst $ maximumBy (compare `on` snd) (M.toList counts) + +computeTreeLoss :: + forall a. + (Columnable a) => + T.Text -> + DataFrame -> + V.Vector Int -> + Tree a -> + Double +computeTreeLoss target df indices tree + | V.null indices = 0 + | otherwise = + case interpret @a df (Col target) of + Left _ -> 1.0 + Right (TColumn col) -> + case toVector @a col of + Left _ -> 1.0 + Right targetVals -> + let + n = V.length indices + errors = + V.length $ + V.filter + (\i -> targetVals V.! i /= predictWithTree @a target df i tree) + indices + in + fromIntegral errors / fromIntegral n + +pruneDead :: Tree a -> Tree a +pruneDead (Leaf v) = Leaf v +pruneDead (Branch cond left right) = let - t = pruneTree trueBranch - f = pruneTree falseBranch + left' = pruneDead left + right' = pruneDead right in - if t == f + Branch cond left' right' + +pruneExpr :: forall a. (Columnable a, Eq a) => Expr a -> Expr a +pruneExpr (If cond trueBranch falseBranch) = + let t = pruneExpr trueBranch + f = pruneExpr falseBranch + in if t == f then t else case (t, f) of - -- Nested simplification: `if C1 then (if C1 then X else Y) else Z` - -- becomes: if C1 then X else Z` - -- Generalize this with hegg later. - (If condInner tInner fInner, _) | cond == condInner -> If cond tInner f - (_, If condInner tInner fInner) | cond == condInner -> If cond t fInner + (If condInner tInner _, _) | cond == condInner -> If cond tInner f + (_, If condInner _ fInner) | cond == condInner -> If cond t fInner _ -> If cond t f -pruneTree (UnaryOp name op e) = UnaryOp name op (pruneTree e) -pruneTree (BinaryOp name op l r) = BinaryOp name op (pruneTree l) (pruneTree r) -pruneTree e = e +pruneExpr (UnaryOp name op e) = UnaryOp name op (pruneExpr e) +pruneExpr (BinaryOp name op l r) = BinaryOp name op (pruneExpr l) (pruneExpr r) +pruneExpr e = e -type CondGen = TreeConfig -> DataFrame -> [Expr Bool] +buildGreedyTree :: + forall a. + (Columnable a) => + TreeConfig -> + Int -> + T.Text -> + [Expr Bool] -> + DataFrame -> + Tree a +buildGreedyTree cfg depth target conds df + | depth <= 0 || nRows df <= minSamplesSplit cfg = + Leaf (majorityValue @a target df) + | otherwise = + case findBestGreedySplit @a cfg target conds df of + Nothing -> Leaf (majorityValue @a target df) + Just bestCond -> + let (dfTrue, dfFalse) = partitionDataFrame bestCond df + in if nRows dfTrue < minLeafSize cfg || nRows dfFalse < minLeafSize cfg + then Leaf (majorityValue @a target df) + else + Branch + bestCond + (buildGreedyTree @a cfg (depth - 1) target conds dfTrue) + (buildGreedyTree @a cfg (depth - 1) target conds dfFalse) -numericConditions :: CondGen +findBestGreedySplit :: + forall a. + (Columnable a) => + TreeConfig -> T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool) +findBestGreedySplit cfg target conds df = + let + initialImpurity = calculateGini @a target df + calculateComplexity c = complexityPenalty (synthConfig cfg) * fromIntegral (eSize c) + + evalGain :: Expr Bool -> (Double, Int) + evalGain cond = + let (t, f) = partitionDataFrame cond df + n = fromIntegral @Int @Double (nRows df) + weightT = fromIntegral @Int @Double (nRows t) / n + weightF = fromIntegral @Int @Double (nRows f) / n + newImpurity = + weightT * calculateGini @a target t + + weightF * calculateGini @a target f + in ( (initialImpurity - newImpurity) - calculateComplexity cond + , negate (eSize cond) + ) + + validConds = + filter + ( \c -> + let (t, f) = partitionDataFrame c df + in nRows t >= minLeafSize cfg && nRows f >= minLeafSize cfg + ) + conds + + sortedConditions = + map fst $ + take + (expressionPairs cfg) + ( filter + (\(c, v) -> ((> negate (calculateComplexity c)) . fst) v) + (sortBy (flip compare `on` snd) (map (\c -> (c, evalGain c)) validConds)) + ) + in + if null sortedConditions + then Nothing + else + Just $ + maximumBy + (compare `on` evalGain) + ( boolExprs + df + sortedConditions + sortedConditions + 0 + (boolExpansion (synthConfig cfg)) + ) + +numericConditions :: TreeConfig -> DataFrame -> [Expr Bool] numericConditions = generateNumericConds -generateNumericConds :: - TreeConfig -> DataFrame -> [Expr Bool] +generateNumericConds :: TreeConfig -> DataFrame -> [Expr Bool] generateNumericConds cfg df = do expr <- numericExprsWithTerms (synthConfig cfg) df let thresholds = map (\p -> percentile p expr df) (percentiles cfg) @@ -167,8 +562,7 @@ , expr .> F.lit threshold ] -numericExprsWithTerms :: - SynthConfig -> DataFrame -> [Expr Double] +numericExprsWithTerms :: SynthConfig -> DataFrame -> [Expr Double] numericExprsWithTerms cfg df = concatMap (numericExprs cfg df [] 0) [0 .. maxExprDepth cfg] @@ -230,18 +624,14 @@ genConds :: T.Text -> [Expr Bool] genConds colName = case unsafeGetColumn colName df of (BoxedColumn (col :: V.Vector a)) -> - let - percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99] - in - map (Col @a colName .==) percentiles + let ps = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99] + in map (Col @a colName .==) ps (OptionalColumn (col :: V.Vector (Maybe a))) -> case sFloating @a of STrue -> - let - doubleCol = + let doubleCol = VU.convert (V.map fromJust (V.filter isJust (V.map (fmap (realToFrac @a @Double)) col))) - in - zipWith + in zipWith ($) [ (Col @(Maybe a) colName .==) , (Col @(Maybe a) colName .<=) @@ -249,21 +639,15 @@ ] ( Lit Nothing : map - ( Lit - . Just - . realToFrac - . (`percentile'` doubleCol) - ) + (Lit . Just . realToFrac . (`percentile'` doubleCol)) (percentiles cfg) ) SFalse -> case sIntegral @a of STrue -> - let - doubleCol = + let doubleCol = VU.convert (V.map fromJust (V.filter isJust (V.map (fmap (fromIntegral @a @Double)) col))) - in - zipWith + in zipWith ($) [ (Col @(Maybe a) colName .==) , (Col @(Maybe a) colName .<=) @@ -271,18 +655,15 @@ ] ( Lit Nothing : map - ( Lit - . Just - . round - . (`percentile'` doubleCol) - ) + (Lit . Just . round . (`percentile'` doubleCol)) (percentiles cfg) ) SFalse -> map ((Col @(Maybe a) colName .==) . Lit . (`percentileOrd'` col)) [1, 25, 75, 99] - (UnboxedColumn (col :: VU.Vector a)) -> [] + (UnboxedColumn (_ :: VU.Vector a)) -> [] + columnConds = concatMap colConds @@ -297,12 +678,13 @@ ] where colConds (!l, !r) = case (unsafeGetColumn l df, unsafeGetColumn r df) of - (BoxedColumn (col1 :: V.Vector a), BoxedColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of - Nothing -> [] - Just Refl -> [Col @a l .== Col @a r] - (UnboxedColumn (col1 :: VU.Vector a), UnboxedColumn (col2 :: VU.Vector b)) -> [] - ( OptionalColumn (col1 :: V.Vector (Maybe a)) - , OptionalColumn (col2 :: V.Vector (Maybe b)) + (BoxedColumn (col1 :: V.Vector a), BoxedColumn (_ :: V.Vector b)) -> + case testEquality (typeRep @a) (typeRep @b) of + Nothing -> [] + Just Refl -> [Col @a l .== Col @a r] + (UnboxedColumn (_ :: VU.Vector a), UnboxedColumn (_ :: VU.Vector b)) -> [] + ( OptionalColumn (_ :: V.Vector (Maybe a)) + , OptionalColumn (_ :: V.Vector (Maybe b)) ) -> case testEquality (typeRep @a) (typeRep @b) of Nothing -> [] Just Refl -> case testEquality (typeRep @a) (typeRep @T.Text) of @@ -315,67 +697,7 @@ partitionDataFrame :: Expr Bool -> DataFrame -> (DataFrame, DataFrame) partitionDataFrame cond df = (filterWhere cond df, filterWhere (F.not cond) df) -findBestSplit :: - forall a. - (Columnable a) => - TreeConfig -> T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool) -findBestSplit cfg target conds df = - let - initialImpurity = calculateGini @a target df - calculateComplexity c = complexityPenalty (synthConfig cfg) * fromIntegral (eSize c) - evalGain :: Expr Bool -> (Double, Int) - evalGain cond = - let (t, f) = partitionDataFrame cond df - n = fromIntegral @Int @Double (nRows df) - weightT = fromIntegral @Int @Double (nRows t) / n - weightF = fromIntegral @Int @Double (nRows f) / n - newImpurity = - (weightT * calculateGini @a target t) - + (weightF * calculateGini @a target f) - in ( (initialImpurity - newImpurity) - - calculateComplexity cond - , negate (eSize cond) - ) - - validConds = - filter - ( \c -> - let - (t, f) = partitionDataFrame c df - in - nRows t >= minLeafSize cfg && nRows f >= minLeafSize cfg - ) - conds - sortedConditions = - map fst $ - take - (expressionPairs cfg) - ( filter - ( \(c, v) -> - ((> negate (calculateComplexity c)) . fst) - v - ) - (sortBy (flip compare `on` snd) (map (\c -> (c, evalGain c)) validConds)) - ) - in - if null sortedConditions - then Nothing - else - Just $ - maximumBy - (compare `on` evalGain) - ( boolExprs - df - sortedConditions - sortedConditions - 0 - (boolExpansion (synthConfig cfg)) - ) - -calculateGini :: - forall a. - (Columnable a) => - T.Text -> DataFrame -> Double +calculateGini :: forall a. (Columnable a) => T.Text -> DataFrame -> Double calculateGini target df = let n = fromIntegral $ nRows df counts = getCounts @a target df @@ -383,20 +705,14 @@ probs = map (\c -> (fromIntegral c + 1) / (n + numClasses)) (M.elems counts) in if n == 0 then 0 else 1 - sum (map (^ 2) probs) -majorityValue :: - forall a. - (Columnable a) => - T.Text -> DataFrame -> a +majorityValue :: forall a. (Columnable a) => T.Text -> DataFrame -> a majorityValue target df = let counts = getCounts @a target df in if M.null counts then error "Empty DataFrame in leaf" else fst $ maximumBy (compare `on` snd) (M.toList counts) -getCounts :: - forall a. - (Columnable a) => - T.Text -> DataFrame -> M.Map a Int +getCounts :: forall a. (Columnable a) => T.Text -> DataFrame -> M.Map a Int getCounts target df = case interpret @a df (Col target) of Left e -> throw e @@ -404,3 +720,42 @@ case toVector @a col of Left e -> throw e Right vals -> foldl' (\acc x -> M.insertWith (+) x 1 acc) M.empty (V.toList vals) + +percentile :: Int -> Expr Double -> DataFrame -> Double +percentile p expr df = + case interpret @Double df expr of + Left _ -> 0 + Right (TColumn col) -> + case toVector @Double col of + Left _ -> 0 + Right vals -> + let sorted = V.fromList $ sort $ V.toList vals + n = V.length sorted + idx = min (n - 1) $ max 0 $ (p * n) `div` 100 + in if n == 0 then 0 else sorted V.! idx + +buildTree :: + forall a. + (Columnable a) => + TreeConfig -> + Int -> + T.Text -> + [Expr Bool] -> + DataFrame -> + Expr a +buildTree cfg depth target conds df = + let + tree = buildGreedyTree @a cfg depth target conds df + indices = V.enumFromN 0 (nRows df) + optimized = taoOptimize @a cfg target conds df indices tree + in + pruneExpr (treeToExpr optimized) + +findBestSplit :: + forall a. + (Columnable a) => + TreeConfig -> T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool) +findBestSplit = findBestGreedySplit @a + +pruneTree :: forall a. (Columnable a, Eq a) => Expr a -> Expr a +pruneTree = pruneExpr diff --git a/src/DataFrame/IO/CSV.hs b/src/DataFrame/IO/CSV.hs --- a/src/DataFrame/IO/CSV.hs +++ b/src/DataFrame/IO/CSV.hs @@ -167,6 +167,8 @@ -- ^ Character that separates column values. , numColumns :: Maybe Int -- ^ Number of columns to read. + , missingIndicators :: [T.Text] + -- ^ Values that should be read as `Nothing`. } shouldInferFromSample :: TypeSpec -> Bool @@ -190,6 +192,7 @@ , dateFormat = "%Y-%m-%d" , columnSeparator = ',' , numColumns = Nothing + , missingIndicators = [] } {- | Read CSV file from path and load it into a dataframe. @@ -236,7 +239,8 @@ readSeparated :: ReadOptions -> FilePath -> IO DataFrame readSeparated !opts !path = do let sep = columnSeparator opts - csvData <- BL.readFile path + let stripUtf8Bom bs = fromMaybe bs (BL.stripPrefix "\xEF\xBB\xBF" bs) + csvData <- stripUtf8Bom <$> BL.readFile path let decodeOpts = Csv.defaultDecodeOptions{Csv.decDelimiter = fromIntegral (ord sep)} let stream = CsvStream.decodeWith decodeOpts Csv.NoHeader csvData @@ -263,7 +267,11 @@ (sampleRow, _) <- peekStream rowsToProcess builderCols <- initializeColumns (V.toList sampleRow) opts - processStream rowsToProcess builderCols (numColumns opts) + processStream + (missingIndicators opts) + rowsToProcess + builderCols + (numColumns opts) frozenCols <- V.fromList <$> mapM freezeBuilderColumn builderCols let numRows = maybe 0 columnLength (frozenCols V.!? 0) @@ -278,6 +286,7 @@ if shouldInferFromSample (typeSpec opts) then parseDefaults + (missingIndicators opts) (typeInferenceSampleSize (typeSpec opts)) (safeRead opts) (dateFormat opts) @@ -305,17 +314,20 @@ Nothing -> BuilderText <$> newPagedVector <*> pure validityRef processStream :: + [T.Text] -> CsvStream.Records (V.Vector BL.ByteString) -> [BuilderColumn] -> Maybe Int -> IO () -processStream _ _ (Just 0) = return () -processStream (Cons (Right row) rest) cols n = processRow row cols >> processStream rest cols (fmap (flip (-) 1) n) -processStream (Cons (Left err) _) _ _ = error ("CSV Parse Error: " ++ err) -processStream (Nil _ _) _ _ = return () +processStream _ _ _ (Just 0) = return () +processStream missing (Cons (Right row) rest) cols n = + processRow missing row cols + >> processStream missing rest cols (fmap (flip (-) 1) n) +processStream missing (Cons (Left err) _) _ _ = error ("CSV Parse Error: " ++ err) +processStream missing (Nil _ _) _ _ = return () -processRow :: V.Vector BL.ByteString -> [BuilderColumn] -> IO () -processRow !vals !cols = V.zipWithM_ processValue vals (V.fromList cols) +processRow :: [T.Text] -> V.Vector BL.ByteString -> [BuilderColumn] -> IO () +processRow missing !vals !cols = V.zipWithM_ processValue vals (V.fromList cols) where processValue !bs !col = do let bs' = BL.toStrict bs @@ -328,7 +340,7 @@ Nothing -> appendPagedUnboxedVector gv 0.0 >> appendPagedUnboxedVector valid 0 BuilderText gv valid -> do let !val = T.strip (TE.decodeUtf8Lenient bs') - if isNullish val + if isNullish val || val `elem` missing then appendPagedVector gv T.empty >> appendPagedUnboxedVector valid 0 else appendPagedVector gv val >> appendPagedUnboxedVector valid 1 @@ -428,11 +440,9 @@ ++ "the other columns at index " ++ show i go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of - Just e -> textRep : acc - where - textRep = case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> fromMaybe "Nothing" e - Nothing -> (T.pack . show) e + Just e -> case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> fromMaybe T.empty e : acc + Nothing -> maybe T.empty (T.pack . show) e : acc Nothing -> error $ "Column " diff --git a/src/DataFrame/IO/Parquet/Thrift.hs b/src/DataFrame/IO/Parquet/Thrift.hs --- a/src/DataFrame/IO/Parquet/Thrift.hs +++ b/src/DataFrame/IO/Parquet/Thrift.hs @@ -1,4 +1,7 @@ +{-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE GADTs #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE StrictData #-} {-# LANGUAGE TypeApplications #-} @@ -13,9 +16,20 @@ import qualified Data.Map as M import Data.Maybe import qualified Data.Text as T +import Data.Typeable (Typeable) +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU import Data.Word import DataFrame.IO.Parquet.Binary import DataFrame.IO.Parquet.Types +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.DataFrame (DataFrame, unsafeGetColumn) +import qualified DataFrame.Operations.Core as DI +import Type.Reflection ( + eqTypeRep, + typeRep, + (:~~:) (HRefl), + ) data SchemaElement = SchemaElement { elementName :: T.Text @@ -31,6 +45,39 @@ } deriving (Show, Eq) +createParquetSchema :: DataFrame -> [SchemaElement] +createParquetSchema df = schemaDef : map toSchemaElement (DI.columnNames df) + where + -- The schema always contains an initial element + -- indicating the group of fields. + schemaDef = + SchemaElement + { elementName = "schema" + , elementType = STOP + , typeLength = 0 + , numChildren = fromIntegral (snd (DI.dimensions df)) + , fieldId = -1 + , repetitionType = UNKNOWN_REPETITION_TYPE + , convertedType = 0 + , scale = 0 + , precision = 0 + , logicalType = LOGICAL_TYPE_UNKNOWN + } + toSchemaElement colName = + let + colType :: TType + colType = case unsafeGetColumn colName df of + (DI.BoxedColumn (col :: V.Vector a)) -> haskellToTType @a + (DI.UnboxedColumn (col :: VU.Vector a)) -> haskellToTType @a + (DI.OptionalColumn (col :: V.Vector (Maybe a))) -> haskellToTType @a + lType = + if DI.hasElemType @T.Text (unsafeGetColumn colName df) + || DI.hasElemType @(Maybe T.Text) (unsafeGetColumn colName df) + then STRING_TYPE + else LOGICAL_TYPE_UNKNOWN + in + SchemaElement colName colType 0 0 (-1) OPTIONAL 0 0 0 lType + data KeyValue = KeyValue { key :: String , value :: String @@ -67,6 +114,29 @@ | STRUCT | UUID deriving (Show, Eq) + +haskellToTType :: forall a. (Typeable a) => TType +haskellToTType + | is @Bool = BOOL + | is @Int8 = BYTE + | is @Word8 = BYTE + | is @Int16 = I16 + | is @Word16 = I16 + | is @Int32 = I32 + | is @Word32 = I32 + | is @Int64 = I64 + | is @Word64 = I64 + | is @Float = FLOAT + | is @Double = DOUBLE + | is @String = STRING + | is @T.Text = STRING + | is @BS.ByteString = STRING + | otherwise = STOP + where + is :: forall x. (Typeable x) => Bool + is = case eqTypeRep (typeRep @a) (typeRep @x) of + Just HRefl -> True + Nothing -> False defaultMetadata :: FileMetadata defaultMetadata = diff --git a/src/DataFrame/IO/Unstable/CSV.hs b/src/DataFrame/IO/Unstable/CSV.hs --- a/src/DataFrame/IO/Unstable/CSV.hs +++ b/src/DataFrame/IO/Unstable/CSV.hs @@ -124,6 +124,7 @@ then typeInferenceSampleSize (typeSpec opts) else 0 in parseFromExamples + (missingIndicators opts) n (safeRead opts) (dateFormat opts) diff --git a/src/DataFrame/Internal/Column.hs b/src/DataFrame/Internal/Column.hs --- a/src/DataFrame/Internal/Column.hs +++ b/src/DataFrame/Internal/Column.hs @@ -28,6 +28,7 @@ import Control.Exception (throw) import Control.Monad.ST (runST) +import Data.Kind (Type) import Data.Maybe import Data.Type.Equality (TestEquality (..)) import DataFrame.Errors @@ -139,20 +140,18 @@ Just Refl -> a == b (==) _ _ = False +-- Generalised LEQ that does reflection. +generalLEQ :: + forall a b. (Typeable a, Typeable b, Ord a, Ord b) => a -> b -> Bool +generalLEQ x y = case testEquality (typeRep @a) (typeRep @b) of + Nothing -> False + Just Refl -> x <= y + instance Ord Column where (<=) :: Column -> Column -> Bool - (<=) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) = - case testEquality (typeRep @t1) (typeRep @t2) of - Nothing -> False - Just Refl -> a <= b - (<=) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) = - case testEquality (typeRep @t1) (typeRep @t2) of - Nothing -> False - Just Refl -> a <= b - (<=) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) = - case testEquality (typeRep @t1) (typeRep @t2) of - Nothing -> False - Just Refl -> a <= b + (<=) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) = generalLEQ a b + (<=) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) = generalLEQ a b + (<=) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) = generalLEQ a b (<=) _ _ = False {- | A class for converting a vector to a column of the appropriate type. @@ -239,63 +238,71 @@ [a] -> Column fromList = toColumnRep @(KindOf a) . VB.fromList +throwTypeMismatch :: + forall (a :: Type) (b :: Type). + (Typeable a, Typeable b) => Either DataFrameException Column +throwTypeMismatch = + Left $ + TypeMismatchException + MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "mapColumn" + , errorColumnName = Nothing + } + -- | An internal function to map a function over the values of a column. mapColumn :: forall b c. - ( Columnable b - , Columnable c - , UnboxIf c - ) => - (b -> c) -> - Column -> - Either DataFrameException Column + (Columnable b, Columnable c) => + (b -> c) -> Column -> Either DataFrameException Column mapColumn f = \case - BoxedColumn (col :: VB.Vector a) - | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Right (fromVector @c (VB.map f col)) - | otherwise -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "mapColumn" - , errorColumnName = Nothing - } - ) - OptionalColumn (col :: VB.Vector a) - | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Right (fromVector @c (VB.map f col)) - | otherwise -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "mapColumn" - , errorColumnName = Nothing - } - ) - UnboxedColumn (col :: VU.Vector a) - | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - Right $ case sUnbox @c of - STrue -> UnboxedColumn (VU.map f col) - SFalse -> fromVector @c (VB.generate (VU.length col) (f . VU.unsafeIndex col)) - | otherwise -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "mapColumn" - , errorColumnName = Nothing - } - ) + BoxedColumn (col :: VB.Vector a) -> run col + OptionalColumn (col :: VB.Vector a) -> run col + UnboxedColumn (col :: VU.Vector a) -> runUnboxed col + where + run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column + run col = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right (fromVector @c (VB.map f col)) + Nothing -> throwTypeMismatch @a @b + + runUnboxed :: + forall a. + (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column + runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right $ case sUnbox @c of + STrue -> UnboxedColumn (VU.map f col) + SFalse -> fromVector @c (VB.generate (VU.length col) (f . VU.unsafeIndex col)) + Nothing -> throwTypeMismatch @a @b {-# SPECIALIZE mapColumn :: (Double -> Double) -> Column -> Either DataFrameException Column #-} {-# INLINEABLE mapColumn #-} +-- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column. +imapColumn :: + forall b c. + (Columnable b, Columnable c) => + (Int -> b -> c) -> Column -> Either DataFrameException Column +imapColumn f = \case + BoxedColumn (col :: VB.Vector a) -> run col + OptionalColumn (col :: VB.Vector a) -> run col + UnboxedColumn (col :: VU.Vector a) -> runUnboxed col + where + run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column + run col = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right (fromVector @c (VB.imap f col)) + Nothing -> throwTypeMismatch @a @b + + runUnboxed :: + forall a. + (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column + runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right $ case sUnbox @c of + STrue -> UnboxedColumn (VU.imap f col) + SFalse -> BoxedColumn (VB.imap f (VG.convert col)) + Nothing -> throwTypeMismatch @a @b + -- | O(1) Gets the number of elements in the column. columnLength :: Column -> Int columnLength (BoxedColumn xs) = VG.length xs @@ -373,158 +380,46 @@ (a -> Bool) -> Column -> Either DataFrameException (VU.Vector Int) -findIndices pred (BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> pure $ VG.convert (VG.findIndices pred column) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "findIndices" - , errorColumnName = Nothing - } - ) -findIndices pred (UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> pure $ VG.findIndices pred column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "findIndices" - , errorColumnName = Nothing - } - ) -findIndices pred (OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> pure $ VG.convert (VG.findIndices pred column) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "findIndices" - , errorColumnName = Nothing - } - ) - --- | An internal function that returns a vector of how indexes change after a column is sorted. -sortedIndexes :: Bool -> Column -> VU.Vector Int -sortedIndexes asc (BoxedColumn column) = runST $ do - withIndexes <- VG.thaw $ VG.indexed column - VA.sortBy - (\(a, b) (a', b') -> (if asc then compare else flip compare) b b') - withIndexes - sorted <- VG.unsafeFreeze withIndexes - return $ VU.generate (VG.length column) (\i -> fst (sorted VG.! i)) -sortedIndexes asc (UnboxedColumn column) = runST $ do - withIndexes <- VG.thaw $ VG.indexed column - VA.sortBy - (\(a, b) (a', b') -> (if asc then compare else flip compare) b b') - withIndexes - sorted <- VG.unsafeFreeze withIndexes - return $ VU.generate (VG.length column) (\i -> fst (sorted VG.! i)) -sortedIndexes asc (OptionalColumn column) = runST $ do - withIndexes <- VG.thaw $ VG.indexed column - VA.sortBy - (\(a, b) (a', b') -> (if asc then compare else flip compare) b b') - withIndexes - sorted <- VG.unsafeFreeze withIndexes - return $ VU.generate (VG.length column) (\i -> fst (sorted VG.! i)) -{-# INLINE sortedIndexes #-} - --- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column. -imapColumn :: - forall b c. - (Columnable b, Columnable c) => - (Int -> b -> c) -> Column -> Either DataFrameException Column -imapColumn f = \case - BoxedColumn (col :: VB.Vector a) - | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - pure (fromVector @c (VB.imap f col)) - | otherwise -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "imapColumn" - , errorColumnName = Nothing - } - ) - UnboxedColumn (col :: VU.Vector a) - | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - pure $ - case sUnbox @c of - STrue -> UnboxedColumn (VU.imap f col) - SFalse -> fromVector @c (VB.imap f (VB.convert col)) - | otherwise -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "imapColumn" - , errorColumnName = Nothing - } - ) - OptionalColumn (col :: VB.Vector a) - | Just Refl <- testEquality (typeRep @a) (typeRep @b) -> - pure (fromVector @c (VB.imap f col)) - | otherwise -> +findIndices pred = \case + BoxedColumn (v :: VB.Vector b) -> run v VG.convert + OptionalColumn (v :: VB.Vector b) -> run v VG.convert + UnboxedColumn (v :: VU.Vector b) -> run v id + where + run :: + forall b v. + (Typeable b, VG.Vector v b, VG.Vector v Int) => + v b -> + (v Int -> VU.Vector Int) -> + Either DataFrameException (VU.Vector Int) + run column finalize = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right . finalize $ VG.findIndices pred column + Nothing -> Left $ TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "imapColumn" + MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "findIndices" , errorColumnName = Nothing } - ) --- | Filter column with index. -ifilterColumn :: - forall a. - (Columnable a) => - (Int -> a -> Bool) -> Column -> Either DataFrameException Column -ifilterColumn f c@(BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> pure $ BoxedColumn $ VG.ifilter f column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "ifilterColumn" - , errorColumnName = Nothing - } - ) -ifilterColumn f c@(UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> pure $ UnboxedColumn $ VG.ifilter f column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "ifilterColumn" - , errorColumnName = Nothing - } - ) -ifilterColumn f c@(OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> pure $ OptionalColumn $ VG.ifilter f column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "ifilterColumn" - , errorColumnName = Nothing - } - ) +-- | An internal function that returns a vector of how indexes change after a column is sorted. +sortedIndexes :: Bool -> Column -> VU.Vector Int +sortedIndexes asc = \case + BoxedColumn column -> sortWorker column + UnboxedColumn column -> sortWorker column + OptionalColumn column -> sortWorker column + where + sortWorker :: + (VG.Vector v a, Ord a, VG.Vector v (Int, a), VG.Vector v Int) => + v a -> VU.Vector Int + sortWorker column = runST $ do + withIndexes <- VG.thaw $ VG.indexed column + let cmp = if asc then compare else flip compare + VA.sortBy (\(_, b) (_, b') -> cmp b b') withIndexes + sorted <- VG.unsafeFreeze withIndexes + return $ VG.convert $ VG.map fst sorted +{-# INLINE sortedIndexes #-} -- | Fold (right) column with index. ifoldrColumn :: @@ -568,48 +463,6 @@ } ) --- | Fold (left) column with index. -ifoldlColumn :: - forall a b. - (Columnable a, Columnable b) => - (b -> Int -> a -> b) -> b -> Column -> Either DataFrameException b -ifoldlColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of - Just Refl -> pure $ VG.ifoldl' f acc column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @d) - , callingFunctionName = Just "ifoldlColumn" - , errorColumnName = Nothing - } - ) -ifoldlColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of - Just Refl -> pure $ VG.ifoldl' f acc column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @d) - , callingFunctionName = Just "ifoldlColumn" - , errorColumnName = Nothing - } - ) -ifoldlColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of - Just Refl -> pure $ VG.ifoldl' f acc column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @d) - , callingFunctionName = Just "ifoldlColumn" - , errorColumnName = Nothing - } - ) - foldlColumn :: forall a b. (Columnable a, Columnable b) => @@ -738,55 +591,6 @@ , errorColumnName = Nothing } ) - --- | Generic reduce function for all Column types. -reduceColumn :: - forall a b. - (Columnable a, Columnable b) => - (a -> b) -> Column -> Either DataFrameException b -{-# SPECIALIZE reduceColumn :: - (VU.Vector (Double, Double) -> Double) -> - Column -> - Either DataFrameException Double - , (VU.Vector Double -> Double) -> Column -> Either DataFrameException Double - #-} -reduceColumn f (BoxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of - Just Refl -> pure $ f column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "reduceColumn" - , errorColumnName = Nothing - } - ) -reduceColumn f (UnboxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of - Just Refl -> pure $ f column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "reduceColumn" - , errorColumnName = Nothing - } - ) -reduceColumn f (OptionalColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of - Just Refl -> pure $ f column - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "reduceColumn" - , errorColumnName = Nothing - } - ) -{-# INLINE reduceColumn #-} -- | An internal, column version of zip. zipColumns :: Column -> Column -> Column diff --git a/src/DataFrame/Lazy/IO/CSV.hs b/src/DataFrame/Lazy/IO/CSV.hs --- a/src/DataFrame/Lazy/IO/CSV.hs +++ b/src/DataFrame/Lazy/IO/CSV.hs @@ -383,11 +383,9 @@ ++ "the other columns at index " ++ show i go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of - Just e -> textRep : acc - where - textRep = case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> fromMaybe "Nothing" e - Nothing -> (T.pack . show) e + Just e -> case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> fromMaybe T.empty e : acc + Nothing -> maybe T.empty (T.pack . show) e : acc Nothing -> error $ "Column " diff --git a/src/DataFrame/Monad.hs b/src/DataFrame/Monad.hs --- a/src/DataFrame/Monad.hs +++ b/src/DataFrame/Monad.hs @@ -14,6 +14,7 @@ import DataFrame.Internal.Expression (Expr (..)) import qualified Data.Text as T +import System.Random -- A re-implementation of the state monad. -- `mtl` might be too heavy a dependency just to get @@ -60,6 +61,12 @@ filterWhereM :: Expr Bool -> FrameM () filterWhereM p = modifyM (D.filterWhere p) + +sampleM :: (RandomGen g) => g -> Double -> FrameM () +sampleM pureGen p = modifyM (D.sample pureGen p) + +takeM :: Int -> FrameM () +takeM n = modifyM (D.take n) filterJustM :: (Columnable a) => Expr (Maybe a) -> FrameM (Expr a) filterJustM (Col name) = FrameM $ \df -> diff --git a/src/DataFrame/Operations/Core.hs b/src/DataFrame/Operations/Core.hs --- a/src/DataFrame/Operations/Core.hs +++ b/src/DataFrame/Operations/Core.hs @@ -864,7 +864,8 @@ This may occur if the column contains non-numeric data or values outside the 'Int' range. -} columnAsIntVector :: - Expr Int -> DataFrame -> Either DataFrameException (VU.Vector Int) + (Columnable a, Num a) => + Expr a -> DataFrame -> Either DataFrameException (VU.Vector Int) columnAsIntVector (Col name) df = case getColumn name df of Just col -> toIntVector col Nothing -> @@ -880,7 +881,8 @@ This may occur if the column contains non-numeric data. -} columnAsDoubleVector :: - Expr Double -> DataFrame -> Either DataFrameException (VU.Vector Double) + (Columnable a, Num a) => + Expr a -> DataFrame -> Either DataFrameException (VU.Vector Double) columnAsDoubleVector (Col name) df = case getColumn name df of Just col -> toDoubleVector col Nothing -> @@ -896,7 +898,8 @@ This may occur if the column contains non-numeric data. -} columnAsFloatVector :: - Expr Float -> DataFrame -> Either DataFrameException (VU.Vector Float) + (Columnable a, Num a) => + Expr a -> DataFrame -> Either DataFrameException (VU.Vector Float) columnAsFloatVector (Col name) df = case getColumn name df of Just col -> toFloatVector col Nothing -> diff --git a/src/DataFrame/Operations/Subset.hs b/src/DataFrame/Operations/Subset.hs --- a/src/DataFrame/Operations/Subset.hs +++ b/src/DataFrame/Operations/Subset.hs @@ -18,11 +18,23 @@ import Control.Exception (throw) import Data.Function ((&)) -import Data.Maybe (fromJust, fromMaybe, isJust, isNothing) +import Data.Maybe ( + fromJust, + fromMaybe, + isJust, + isNothing, + ) import Data.Type.Equality (TestEquality (..)) -import DataFrame.Errors (DataFrameException (..), TypeErrorContext (..)) +import DataFrame.Errors ( + DataFrameException (..), + TypeErrorContext (..), + ) import DataFrame.Internal.Column -import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn) +import DataFrame.Internal.DataFrame ( + DataFrame (..), + empty, + getColumn, + ) import DataFrame.Internal.Expression import DataFrame.Internal.Interpreter import DataFrame.Operations.Core diff --git a/src/DataFrame/Operations/Typing.hs b/src/DataFrame/Operations/Typing.hs --- a/src/DataFrame/Operations/Typing.hs +++ b/src/DataFrame/Operations/Typing.hs @@ -22,28 +22,35 @@ type DateFormat = String -parseDefaults :: Int -> Bool -> DateFormat -> DataFrame -> DataFrame -parseDefaults n safeRead dateFormat df = df{columns = V.map (parseDefault n safeRead dateFormat) (columns df)} +parseDefaults :: [T.Text] -> Int -> Bool -> DateFormat -> DataFrame -> DataFrame +parseDefaults missing n safeRead dateFormat df = df{columns = V.map (parseDefault missing n safeRead dateFormat) (columns df)} -parseDefault :: Int -> Bool -> DateFormat -> Column -> Column -parseDefault n safeRead dateFormat (BoxedColumn (c :: V.Vector a)) = +parseDefault :: [T.Text] -> Int -> Bool -> DateFormat -> Column -> Column +parseDefault missing n safeRead dateFormat (BoxedColumn (c :: V.Vector a)) = case (typeRep @a) `testEquality` (typeRep @T.Text) of Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of - Just Refl -> parseFromExamples n safeRead dateFormat (V.map T.pack c) + Just Refl -> parseFromExamples missing n safeRead dateFormat (V.map T.pack c) Nothing -> BoxedColumn c - Just Refl -> parseFromExamples n safeRead dateFormat c -parseDefault n safeRead dateFormat (OptionalColumn (c :: V.Vector (Maybe a))) = + Just Refl -> parseFromExamples missing n safeRead dateFormat c +parseDefault missing n safeRead dateFormat (OptionalColumn (c :: V.Vector (Maybe a))) = case (typeRep @a) `testEquality` (typeRep @T.Text) of Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of - Just Refl -> parseFromExamples n safeRead dateFormat (V.map (T.pack . fromMaybe "") c) + Just Refl -> + parseFromExamples + missing + n + safeRead + dateFormat + (V.map (T.pack . fromMaybe "") c) Nothing -> BoxedColumn c - Just Refl -> parseFromExamples n safeRead dateFormat (V.map (fromMaybe "") c) -parseDefault _ _ _ column = column + Just Refl -> parseFromExamples missing n safeRead dateFormat (V.map (fromMaybe "") c) +parseDefault _ _ _ _ column = column -parseFromExamples :: Int -> Bool -> DateFormat -> V.Vector T.Text -> Column -parseFromExamples n safeRead dateFormat cols = +parseFromExamples :: + [T.Text] -> Int -> Bool -> DateFormat -> V.Vector T.Text -> Column +parseFromExamples missing n safeRead dateFormat cols = let - converter = if safeRead then convertNullish else convertOnlyEmpty + converter = if safeRead then convertNullish missing else convertOnlyEmpty examples = V.map converter (V.take n cols) asMaybeText = V.map converter cols in @@ -123,8 +130,8 @@ parsableAsDouble = vecSameConstructor asMaybeText asMaybeDouble parsableAsDate = vecSameConstructor asMaybeText asMaybeDate -convertNullish :: T.Text -> Maybe T.Text -convertNullish v = if isNullish v then Nothing else Just v +convertNullish :: [T.Text] -> T.Text -> Maybe T.Text +convertNullish missing v = if isNullish v || v `elem` missing then Nothing else Just v convertOnlyEmpty :: T.Text -> Maybe T.Text convertOnlyEmpty v = if v == "" then Nothing else Just v diff --git a/src/DataFrame/Synthesis.hs b/src/DataFrame/Synthesis.hs --- a/src/DataFrame/Synthesis.hs +++ b/src/DataFrame/Synthesis.hs @@ -266,7 +266,9 @@ percentiles df = let doubleColumns = - map (either throw id . ((`columnAsDoubleVector` df) . Col)) (D.columnNames df) + map + (either throw id . ((`columnAsDoubleVector` df) . Col @Double)) + (D.columnNames df) in concatMap (\c -> map (Lit . roundTo2SigDigits . (`percentile'` c)) [1, 25, 75, 99]) diff --git a/tests/GenDataFrame.hs b/tests/GenDataFrame.hs new file mode 100644 --- /dev/null +++ b/tests/GenDataFrame.hs @@ -0,0 +1,42 @@ +{-# LANGUAGE GADTs #-} +{-# LANGUAGE TypeApplications #-} + +module GenDataFrame where + +import qualified Data.Map as M +import qualified Data.Text as T +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU +import DataFrame.Internal.Column +import DataFrame.Internal.DataFrame +import Test.QuickCheck + +genColumn :: Int -> Gen Column +genColumn len = + oneof + [ BoxedColumn . V.fromList <$> vectorOf len (arbitrary @Int) + , UnboxedColumn . VU.fromList <$> vectorOf len (arbitrary @Double) + , OptionalColumn . V.fromList <$> vectorOf len (arbitrary @(Maybe Int)) + ] + +genDataFrame :: Gen DataFrame +genDataFrame = do + numRows <- choose (0, 100) + numCols <- choose (0, 10) + colNames <- V.fromList <$> vectorOf numCols genUniqueColName + cols <- V.fromList <$> vectorOf numCols (genColumn numRows) + let indices = M.fromList $ zip (V.toList colNames) [0 ..] + pure $ + DataFrame + { columns = cols + , columnIndices = indices + , dataframeDimensions = (numRows, numCols) + , derivingExpressions = M.empty + } + +genUniqueColName :: Gen T.Text +genUniqueColName = T.pack <$> listOf1 (elements ['a' .. 'z']) + +instance Arbitrary DataFrame where + arbitrary = genDataFrame + shrink df = [] diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -1,4 +1,5 @@ {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} module Main where @@ -12,7 +13,9 @@ import qualified System.Exit as Exit import Data.Time +import GenDataFrame () import Test.HUnit +import Test.QuickCheck import qualified Functions import qualified Operations.Aggregations @@ -27,6 +30,7 @@ import qualified Operations.ReadCsv import qualified Operations.Sort import qualified Operations.Statistics +import qualified Operations.Subset import qualified Operations.Take import qualified Parquet @@ -59,7 +63,7 @@ beforeParse :: [T.Text] beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Bools without missing values as UnboxedColumn of Bools" @@ -74,7 +78,7 @@ beforeParse :: [T.Text] beforeParse = T.pack . show <$> [1 .. 50] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints without missing values as UnboxedColumn of Ints" @@ -91,7 +95,7 @@ T.pack . show <$> [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Doubles without missing values as UnboxedColumn of Doubles" @@ -138,7 +142,7 @@ , "2020-02-26" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Dates without missing values as BoxedColumn of Days" @@ -223,7 +227,7 @@ , "Meowth, that's right!" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Text without missing values as BoxedColumn of Text" @@ -239,7 +243,7 @@ beforeParse :: [T.Text] beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses mixture of Bools and Ints as Text" @@ -466,7 +470,7 @@ , "12.22222235049451" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles" @@ -561,7 +565,7 @@ , "2020-02-20" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints and Dates as BoxedColumn of Texts" @@ -700,7 +704,7 @@ , "12.22222235049451" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Texts and Doubles as BoxedColumn of Texts" @@ -753,7 +757,7 @@ , "Meowth" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Dates and Texts as BoxedColumn of Texts" @@ -770,7 +774,8 @@ beforeParse :: [T.Text] beforeParse = replicate 10 "true" ++ replicate 10 "false" expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Bools without missing values as UnboxedColumn of Bools, when safeRead is off" @@ -887,7 +892,8 @@ , "50" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints without missing values as UnboxedColumn of Ints, when safeRead is off" @@ -1014,7 +1020,8 @@ , "12.22222235049451" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Doubles without missing values as UnboxedColumn of Doubles, when safeRead is off" @@ -1061,7 +1068,8 @@ , "2020-02-26" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Dates without missing values as BoxedColumn of Days" @@ -1146,7 +1154,8 @@ , "Meowth, that's right!" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Text without missing values as BoxedColumn of Text" @@ -1161,7 +1170,8 @@ beforeParse :: [T.Text] beforeParse = replicate 10 "" ++ replicate 10 "true" ++ replicate 10 "false" expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Bools and empty Strings as OptionalColumn of Bools, when safeRead is off" @@ -1288,7 +1298,8 @@ , "50" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is off" @@ -1425,7 +1436,8 @@ , "12.22222235049451" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" @@ -1482,7 +1494,8 @@ , "" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead off" @@ -1581,7 +1594,8 @@ , "Meowth, that's right!" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" @@ -1596,7 +1610,8 @@ beforeParse :: [T.Text] beforeParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False" expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Bools with nullish values as BoxedColumn of Texts, when safeRead is off" @@ -1723,7 +1738,8 @@ , "50" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints with nullish values as BoxedColumn of Texts, when safeRead is off" @@ -1860,7 +1876,8 @@ , "12.22222235049451" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" @@ -1999,7 +2016,8 @@ , "" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Texts, when safeRead is off" @@ -2104,7 +2122,8 @@ , "N/A" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" @@ -2120,7 +2139,7 @@ beforeParse :: [T.Text] beforeParse = replicate 10 "" ++ replicate 10 "true" expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Bools and empty strings as OptionalColumn of Bools, when safeRead is on" @@ -2247,7 +2266,7 @@ , "50" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is on" @@ -2384,7 +2403,7 @@ , "12.22222235049451" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is on" @@ -2441,7 +2460,7 @@ , "" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead on" @@ -2540,7 +2559,7 @@ , "Meowth, that's right!" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead on" @@ -2667,7 +2686,7 @@ , "50" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints with nullish values as OptionalColumn of Ints, when safeRead is on" @@ -2804,7 +2823,7 @@ , "12.03" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" @@ -2943,7 +2962,7 @@ , "" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Ints, when safeRead is on" @@ -3040,7 +3059,7 @@ , "3.14" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints and Doubles with nullish values AND empty strings as OptionalColumn of Doubles, when safeRead is on" @@ -3145,7 +3164,7 @@ , "N/A" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" @@ -3161,7 +3180,7 @@ beforeParse :: [T.Text] beforeParse = "false" : replicate 50 "true" expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example" @@ -3176,7 +3195,7 @@ beforeParse :: [T.Text] beforeParse = "false" : replicate 50 "true" expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example" @@ -3293,7 +3312,7 @@ , "50" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints without missing values as UnboxedColumn of Ints with only one example" @@ -3410,7 +3429,7 @@ , "50" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 25 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 25 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints without missing values as UnboxedColumn of Ints with some examples" @@ -3527,7 +3546,7 @@ , "50" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Ints without missing values as UnboxedColumn of Ints with many examples" @@ -3574,7 +3593,7 @@ , "2020-02-26" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Dates without missing values as BoxedColumn of Days with only one example" @@ -3621,7 +3640,7 @@ , "2020-02-26" ] expected = DI.BoxedColumn $ V.fromList afterParse - actual = D.parseDefault 15 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 15 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses Dates without missing values as BoxedColumn of Days with many examples" @@ -3848,7 +3867,7 @@ , "12.22222235049451" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" @@ -4075,7 +4094,7 @@ , "12.22222235049451" ] expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = D.parseDefault 50 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 50 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" @@ -4322,7 +4341,7 @@ , "12.22222235049451" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" @@ -4570,7 +4589,8 @@ , "12.22222235049451" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" @@ -4738,7 +4758,7 @@ , "12.22222235049451" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with just one example, when safeRead is off" @@ -4906,7 +4926,8 @@ , "12.22222235049451" ] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = + D.parseDefault [] 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase ( assertEqual "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with many examples, when safeRead is off" @@ -4923,7 +4944,7 @@ beforeParse :: [T.Text] beforeParse = replicate 100 "NaN" ++ ["100000"] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4934,7 +4955,7 @@ beforeParse :: [T.Text] beforeParse = replicate 100 "NaN" ++ ["3.14"] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4945,7 +4966,7 @@ beforeParse :: [T.Text] beforeParse = replicate 100 "NaN" ++ ["2024-12-25"] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4956,7 +4977,7 @@ beforeParse :: [T.Text] beforeParse = replicate 100 "NaN" ++ ["2024-12-25", "2024-12-w6"] expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4967,7 +4988,7 @@ beforeParse :: [T.Text] beforeParse = replicate 100 "NaN" expected = DI.OptionalColumn $ V.fromList afterParse - actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse + actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse in TestCase (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -5105,9 +5126,18 @@ ++ Parquet.tests ++ parseTests +isSuccessful :: Result -> Bool +isSuccessful (Success{..}) = True +isSuccessful _ = False + main :: IO () main = do result <- runTestTT tests - if failures result > 0 || errors result > 0 + -- Property tests + propRes <- + mapM + (quickCheckWithResult stdArgs) + Operations.Subset.tests + if failures result > 0 || errors result > 0 || not (all isSuccessful propRes) then Exit.exitFailure else Exit.exitSuccess diff --git a/tests/Operations/Subset.hs b/tests/Operations/Subset.hs new file mode 100644 --- /dev/null +++ b/tests/Operations/Subset.hs @@ -0,0 +1,95 @@ +{-# LANGUAGE GADTs #-} + +module Operations.Subset where + +import DataFrame.Internal.DataFrame +import qualified DataFrame.Operations.Core as D +import qualified DataFrame.Operations.Subset as D +import System.Random + +prop_dropZero :: DataFrame -> Bool +prop_dropZero df = D.drop 0 df == df + +prop_takeZero :: DataFrame -> Bool +prop_takeZero df = fst (dataframeDimensions (D.take 0 df)) == 0 + +prop_takeAll :: DataFrame -> Bool +prop_takeAll df = + let n = fst (dataframeDimensions df) + in D.take n df == df + +prop_dropAll :: DataFrame -> Bool +prop_dropAll df = + let n = fst (dataframeDimensions df) + in fst (dataframeDimensions (D.drop n df)) == 0 + +prop_takeLastZero :: DataFrame -> Bool +prop_takeLastZero df = fst (dataframeDimensions (D.takeLast 0 df)) == 0 + +prop_dropLastZero :: DataFrame -> Bool +prop_dropLastZero df = D.dropLast 0 df == df + +prop_takeLastAll :: DataFrame -> Bool +prop_takeLastAll df = + let n = fst (dataframeDimensions df) + in D.takeLast n df == df + +prop_dropLastAll :: DataFrame -> Bool +prop_dropLastAll df = + let n = fst (dataframeDimensions df) + in fst (dataframeDimensions (D.dropLast n df)) == 0 + +prop_rangeEmpty :: DataFrame -> Bool +prop_rangeEmpty df = + fst (dataframeDimensions (D.range (5, 5) df)) == 0 + +prop_rangeFull :: DataFrame -> Bool +prop_rangeFull df = + let rows = fst (dataframeDimensions df) + in D.range (0, rows) df == df + +prop_selectAll :: DataFrame -> Bool +prop_selectAll df = D.select (D.columnNames df) df == df + +prop_selectEmpty :: DataFrame -> Bool +prop_selectEmpty df = + let result = D.select [] df + in dataframeDimensions result == (0, 0) + +prop_excludeEmpty :: DataFrame -> Bool +prop_excludeEmpty df = D.exclude [] df == df + +prop_excludeAll :: DataFrame -> Bool +prop_excludeAll df = + let result = D.exclude (D.columnNames df) df + in snd (dataframeDimensions result) == 0 + +prop_cubePreservesSmall :: DataFrame -> Bool +prop_cubePreservesSmall df = + let (rows, cols) = dataframeDimensions df + in D.cube (rows + 100, cols + 100) df == df + +prop_sampleEmptyApprox :: DataFrame -> Bool +prop_sampleEmptyApprox df = + let gen = mkStdGen 42 + sampled = D.sample gen 0.0 df + in fst (dataframeDimensions sampled) == 0 + +tests = + [ prop_dropZero + , prop_takeZero + , prop_takeAll + , prop_dropAll + , prop_takeLastZero + , prop_dropLastZero + , prop_takeLastAll + , prop_dropLastAll + , prop_rangeEmpty + , prop_rangeFull + , prop_selectAll + , prop_selectEmpty + , prop_excludeEmpty + , prop_excludeAll + , prop_cubePreservesSmall + , prop_sampleEmptyApprox + ]