dataframe 0.4.0.10 → 0.4.1.0

raw patch · 18 files changed

+1103/−566 lines, 18 filesdep +QuickCheckdep +unixPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: QuickCheck, unix

API changes (from Hackage documentation)

Files

• CHANGELOG.md +7/−0
• app/Main.hs +95/−9
• app/Synthesis.hs +9/−5
• dataframe.cabal +6/−2
• src/DataFrame/DecisionTree.hs +512/−157
• src/DataFrame/IO/CSV.hs +24/−14
• src/DataFrame/IO/Parquet/Thrift.hs +70/−0
• src/DataFrame/IO/Unstable/CSV.hs +1/−0
• src/DataFrame/Internal/Column.hs +102/−298
• src/DataFrame/Lazy/IO/CSV.hs +3/−5
• src/DataFrame/Monad.hs +7/−0
• src/DataFrame/Operations/Core.hs +6/−3
• src/DataFrame/Operations/Subset.hs +15/−3
• src/DataFrame/Operations/Typing.hs +22/−15
• src/DataFrame/Synthesis.hs +3/−1
• tests/GenDataFrame.hs +42/−0
• tests/Main.hs +84/−54
• tests/Operations/Subset.hs +95/−0

@@ -1,5 +1,12 @@ # Revision history for dataframe +## 0.4.1.0+* Improve signal handling of dataframe repl.+* `writeCsv` not correctly writes `Maybe` values (thanks to @mcoady).+* Create a boilerplate package in cache that will be used to start repl.+* Tree implementation is now a TAO tree instead of greedy cart trees.+* Add `sampleM` and `takeM` functions.+ ## 0.4.0.10 * License in cabal was wrong. * Remove ollama-haskell dependencies.

@@ -1,32 +1,51 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE ScopedTypeVariables #-}+ module Main where +import Control.Exception (bracket) import Control.Monad import Data.Time (NominalDiffTime, diffUTCTime, getCurrentTime) import System.Directory (     XdgDirectory (..),     createDirectoryIfMissing,     doesFileExist,+    getCurrentDirectory,     getModificationTime,     getXdgDirectory,  )+import System.Exit (ExitCode (..)) import System.FilePath ((</>)) import System.Process +#ifndef mingw32_HOST_OS+import System.Posix.Signals (Handler (..), installHandler, sigINT)+#endif+ main :: IO () main = do     cacheDir <- getXdgDirectory XdgCache "dataframe_repl"+    currDir <- getCurrentDirectory+    print currDir     createDirectoryIfMissing True cacheDir -    let filepath = cacheDir </> "dataframe.ghci"-    shouldDownload <- needsUpdate filepath+    let ghciScript = cacheDir </> "dataframe.ghci"+        projectFile = cacheDir </> "cabal.project"+        cabalFile = cacheDir </> "df-repl.cabal"+        mainFile = cacheDir </> "Main.hs" +    -- Ensure the directory is a valid Cabal project with a local target to repl into.+    ensureBoilerplate projectFile cabalFile mainFile++    -- Keep your ghci script updated (weekly).+    shouldDownload <- needsUpdate ghciScript     when shouldDownload $ do         putStrLn "\ESC[92mDownloading latest version of dataframe config...\ESC[0m"         output <-             readProcess                 "curl"                 [ "--output"-                , filepath+                , ghciScript                 , "https://raw.githubusercontent.com/mchav/dataframe/refs/heads/main/dataframe.ghci"                 ]                 ""@@ -37,15 +56,82 @@     let command = "cabal"         args =             [ "repl"+            , "exe:df-repl"+            , "--project-file=" ++ projectFile             , "-O2"-            , "--build-depends"-            , "dataframe"-            , "--repl-option=-ghci-script=" ++ filepath+            , "--repl-option=-ghci-script=" ++ ghciScript             ]-    (_, _, _, processHandle) <- createProcess (proc command args) -    exitCode <- waitForProcess processHandle-    pure ()+        baseCp =+            (proc command args)+                { cwd = Just currDir+                , std_in = Inherit+                , std_out = Inherit+                , std_err = Inherit+                }++#ifdef mingw32_HOST_OS+        cp = baseCp { delegate_ctlc = True }+#else+        cp = baseCp+#endif++#ifndef mingw32_HOST_OS+    -- Unix: ignore Ctrl-C in the wrapper so the child handles it.+    bracket+        (installHandler sigINT Ignore Nothing)+        (\old -> installHandler sigINT old Nothing)+        (\_ -> runChild cp)+#else+    -- Windows: delegate Ctrl-C handling to the child.+    runChild cp+#endif++ensureBoilerplate :: FilePath -> FilePath -> FilePath -> IO ()+ensureBoilerplate projectFile cabalFile mainFile = do+    -- cabal.project+    writeIfMissing+        projectFile+        $ unlines+            [ "packages: ."+            ]++    writeIfMissing+        cabalFile+        $ unlines+            [ "cabal-version:      3.0"+            , "name:               df-repl"+            , "version:            0.1.0.0"+            , "build-type:         Simple"+            , ""+            , "executable df-repl"+            , "  main-is:          Main.hs"+            , "  hs-source-dirs:   ."+            , "  default-language: Haskell2010"+            , "  build-depends:    base, dataframe, text, time, random"+            ]++    writeIfMissing+        mainFile+        $ unlines+            [ "module Main where"+            , ""+            , "main :: IO ()"+            , "main = putStrLn \"df-repl\""+            ]++writeIfMissing :: FilePath -> String -> IO ()+writeIfMissing fp contents = do+    exists <- doesFileExist fp+    unless exists $ writeFile fp contents++runChild :: CreateProcess -> IO ()+runChild cp = do+    (_, _, _, ph) <- createProcess cp+    ec <- waitForProcess ph+    case ec of+        ExitSuccess -> pure ()+        ExitFailure n -> fail ("cabal repl failed with exit code " <> show n)  oneWeek :: NominalDiffTime oneWeek = 7 * 24 * 60 * 60

@@ -43,7 +43,7 @@                 (D.nRows train)                 combined                 |> D.filterJust (F.name survived)-                |> D.randomSplit (mkStdGen 4232) 0.8+                |> D.randomSplit (mkStdGen 4232) 0.7         -- Split the test out again.         test' =             D.drop@@ -54,14 +54,18 @@             fitDecisionTree                 ( defaultTreeConfig                     { maxTreeDepth = 5-                    , minSamplesSplit = 25-                    , minLeafSize = 15+                    , minSamplesSplit = 10+                    , minLeafSize = 3+                    , taoIterations = 100                     , synthConfig =                         defaultSynthConfig-                            { complexityPenalty = 0+                            { complexityPenalty = 0.00                             , maxExprDepth = 2                             , disallowedCombinations =-                                [(F.name age, F.name fare)]+                                [ (F.name age, F.name fare)+                                , ("passenger_class", "number_of_siblings_and_spouses")+                                , ("passenger_class", "number_of_parents_and_children")+                                ]                             }                     }                 )

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.4.0.10+version:            0.4.1.0  synopsis: A fast, safe, and intuitive DataFrame library. @@ -157,7 +157,8 @@                       directory >= 1.3.0.0 && < 2,                       filepath >= 1.4 && < 2,                       process >= 1.6 && < 2,-                      time >= 1.12 && < 2+                      time >= 1.12 && < 2,+                      unix >= 2 && < 3     hs-source-dirs:   app     default-language: Haskell2010     ghc-options: -rtsopts -threaded -with-rtsopts=-N@@ -183,6 +184,7 @@     main-is: Main.hs     other-modules: Assertions,                    Functions,+                   GenDataFrame,                    Operations.Aggregations,                    Operations.Apply,                    Operations.Core,@@ -194,6 +196,7 @@                    Operations.Merge,                    Operations.ReadCsv,                    Operations.Sort,+                   Operations.Subset,                    Operations.Statistics,                    Operations.Take,                    Parquet@@ -201,6 +204,7 @@                     dataframe ^>= 0.4,                     directory >= 1.3.0.0 && < 2,                     HUnit ^>= 1.6,+                    QuickCheck >= 2 && < 3,                     random >= 1 && < 2,                     random-shuffle >= 0.0.4 && < 1,                     random >= 1 && < 2,

@@ -1,11 +1,7 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE BangPatterns #-}-{-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-}-{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-}-{-# LANGUAGE MultiParamTypeClasses #-}-{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -14,19 +10,18 @@ import qualified DataFrame.Functions as F import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn)-import DataFrame.Internal.Expression (Expr (..), eSize, getColumns, prettyPrint)+import DataFrame.Internal.Expression (Expr (..), eSize, getColumns) import DataFrame.Internal.Interpreter (interpret) import DataFrame.Internal.Statistics (percentile', percentileOrd') import DataFrame.Internal.Types import DataFrame.Operations.Core (columnNames, nRows)-import DataFrame.Operations.Statistics (percentile) import DataFrame.Operations.Subset (exclude, filterWhere) -import Control.Exception (SomeException, throw, try)+import Control.Exception (throw) import Control.Monad (guard) import Data.Containers.ListUtils (nubOrd) import Data.Function (on)-import Data.List (foldl', maximumBy, sort, sortBy)+import Data.List (foldl', maximumBy, minimumBy, sort, sortBy) import qualified Data.Map.Strict as M import Data.Maybe import qualified Data.Text as T@@ -35,19 +30,17 @@ import qualified Data.Vector.Unboxed as VU import Type.Reflection (typeRep) -import Data.Text.Read import DataFrame.Functions ((./=), (.<), (.<=), (.==), (.>), (.>=))-import Debug.Trace (trace)-import GHC.IO.Unsafe (unsafePerformIO) -data TreeConfig-    = TreeConfig+data TreeConfig = TreeConfig     { maxTreeDepth :: Int     , minSamplesSplit :: Int     , minLeafSize :: Int     , percentiles :: [Int]     , expressionPairs :: Int     , synthConfig :: SynthConfig+    , taoIterations :: Int+    , taoConvergenceTol :: Double     }     deriving (Eq, Show) @@ -83,8 +76,25 @@         , percentiles = [0, 10 .. 100]         , expressionPairs = 10         , synthConfig = defaultSynthConfig+        , taoIterations = 10+        , taoConvergenceTol = 1e-6         } +data Tree a+    = Leaf !a+    | Branch !(Expr Bool) !(Tree a) !(Tree a)+    deriving (Eq, Show)++treeDepth :: Tree a -> Int+treeDepth (Leaf _) = 0+treeDepth (Branch _ l r) = 1 + max (treeDepth l) (treeDepth r)++treeToExpr :: (Columnable a) => Tree a -> Expr a+treeToExpr (Leaf v) = Lit v+treeToExpr (Branch cond left right) =+    F.ifThenElse cond (treeToExpr left) (treeToExpr right)++-- | Fit a TAO decision tree fitDecisionTree ::     forall a.     (Columnable a) =>@@ -93,70 +103,455 @@     DataFrame ->     Expr a fitDecisionTree cfg (Col target) df =-    buildTree @a-        cfg-        (maxTreeDepth cfg)-        target-        ( nubOrd $-            numericConditions cfg (exclude [target] df)-                ++ generateConditionsOld cfg (exclude [target] df)-        )-        df+    let+        conds =+            nubOrd $+                numericConditions cfg (exclude [target] df)+                    ++ generateConditionsOld cfg (exclude [target] df)++        initialTree = buildGreedyTree @a cfg (maxTreeDepth cfg) target conds df++        indices = V.enumFromN 0 (nRows df)++        optimizedTree = taoOptimize @a cfg target conds df indices initialTree+     in+        pruneExpr (treeToExpr optimizedTree) fitDecisionTree _ expr _ = error $ "Cannot create tree for compound expression: " ++ show expr -buildTree ::+taoOptimize ::     forall a.     (Columnable a) =>     TreeConfig ->+    T.Text -> -- Target column name+    [Expr Bool] -> -- Candidate conditions+    DataFrame -> -- Full dataset+    V.Vector Int -> -- Indices of points reaching the root+    Tree a -> -- Current tree+    Tree a+taoOptimize cfg target conds df rootIndices initialTree =+    go 0 initialTree (computeTreeLoss @a target df rootIndices initialTree)+  where+    go :: Int -> Tree a -> Double -> Tree a+    go iter tree prevLoss+        | iter >= taoIterations cfg = pruneDead tree+        | otherwise =+            let+                tree' = taoIteration @a cfg target conds df rootIndices tree++                newLoss = computeTreeLoss @a target df rootIndices tree'+                improvement = prevLoss - newLoss+             in+                if improvement < taoConvergenceTol cfg+                    then pruneDead tree'+                    else go (iter + 1) tree' newLoss++taoIteration ::+    forall a.+    (Columnable a) =>+    TreeConfig ->+    T.Text ->+    [Expr Bool] ->+    DataFrame ->+    V.Vector Int ->+    Tree a ->+    Tree a+taoIteration cfg target conds df rootIndices tree =+    let depth = treeDepth tree+     in foldl'+            (optimizeDepthLevel @a cfg target conds df rootIndices)+            tree+            [depth, depth - 1 .. 0] -- Bottom to top++optimizeDepthLevel ::+    forall a.+    (Columnable a) =>+    TreeConfig ->+    T.Text ->+    [Expr Bool] ->+    DataFrame ->+    V.Vector Int ->+    Tree a ->+    Int -> -- Target depth+    Tree a+optimizeDepthLevel cfg target conds df rootIndices tree = optimizeAtDepth @a cfg target conds df rootIndices tree 0++optimizeAtDepth ::+    forall a.+    (Columnable a) =>+    TreeConfig ->+    T.Text ->+    [Expr Bool] ->+    DataFrame ->+    V.Vector Int ->+    Tree a ->     Int ->+    Int ->+    Tree a+optimizeAtDepth cfg target conds df indices tree currentDepth targetDepth+    | currentDepth == targetDepth =+        optimizeNode @a cfg target conds df indices tree+    | otherwise = case tree of+        Leaf v -> Leaf v+        Branch cond left right ->+            let+                (indicesL, indicesR) = partitionIndices cond df indices+                left' =+                    optimizeAtDepth @a+                        cfg+                        target+                        conds+                        df+                        indicesL+                        left+                        (currentDepth + 1)+                        targetDepth+                right' =+                    optimizeAtDepth @a+                        cfg+                        target+                        conds+                        df+                        indicesR+                        right+                        (currentDepth + 1)+                        targetDepth+             in+                Branch cond left' right'++optimizeNode ::+    forall a.+    (Columnable a) =>+    TreeConfig ->     T.Text ->     [Expr Bool] ->     DataFrame ->-    Expr a-buildTree cfg depth target conds df-    | depth <= 0 || nRows df <= minSamplesSplit cfg =-        Lit (majorityValue @a target df)+    V.Vector Int ->+    Tree a ->+    Tree a+optimizeNode cfg target conds df indices tree+    | V.null indices = tree+    | otherwise = case tree of+        Leaf _ -> Leaf (majorityValueFromIndices @a target df indices)+        Branch oldCond left right ->+            let+                newCond = findBestSplitTAO @a cfg target conds df indices left right oldCond++                (newIndicesL, newIndicesR) = partitionIndices newCond df indices+             in+                if V.length newIndicesL < minLeafSize cfg+                    || V.length newIndicesR < minLeafSize cfg+                    then Leaf (majorityValueFromIndices @a target df indices)+                    else Branch newCond left right++findBestSplitTAO ::+    forall a.+    (Columnable a) =>+    TreeConfig ->+    T.Text ->+    [Expr Bool] ->+    DataFrame ->+    V.Vector Int ->+    Tree a -> -- Left subtree (FIXED)+    Tree a -> -- Right subtree (FIXED)+    Expr Bool -> -- Current condition (fallback)+    Expr Bool+findBestSplitTAO cfg target conds df indices leftTree rightTree currentCond+    | V.null indices = currentCond+    | null validConds = currentCond     | otherwise =-        case findBestSplit @a cfg target conds df of-            Nothing -> Lit (majorityValue @a target df)-            Just bestCond ->-                let (dfTrue, dfFalse) = partitionDataFrame bestCond df-                 in if nRows dfTrue == 0 || nRows dfFalse == 0-                        then Lit (majorityValue @a target df)-                        else-                            pruneTree-                                ( F.ifThenElse-                                    bestCond-                                    (buildTree @a cfg (depth - 1) target conds dfTrue)-                                    (buildTree @a cfg (depth - 1) target conds dfFalse)-                                )+        let+            carePoints = identifyCarePoints @a target df indices leftTree rightTree+         in+            if null carePoints+                then currentCond+                else+                    let+                        evalSplit :: Expr Bool -> Int+                        evalSplit cond = countCarePointErrors cond df carePoints -pruneTree :: forall a. (Columnable a, Eq a) => Expr a -> Expr a-pruneTree (If cond trueBranch falseBranch) =+                        evalWithPenalty c =+                            let errors = evalSplit c+                                penalty =+                                    floor+                                        ( complexityPenalty (synthConfig cfg)+                                            * fromIntegral (eSize c)+                                        )+                             in errors + penalty++                        sortedConds =+                            take (expressionPairs cfg) $+                                sortBy (compare `on` evalWithPenalty) validConds++                        expandedConds =+                            boolExprs+                                df+                                sortedConds+                                sortedConds+                                0+                                (boolExpansion (synthConfig cfg))+                     in+                        if null expandedConds+                            then currentCond+                            else minimumBy (compare `on` evalWithPenalty) expandedConds+  where+    validConds = filter isValidSplit conds+    isValidSplit c =+        let (t, f) = partitionIndices c df indices+         in V.length t >= minLeafSize cfg && V.length f >= minLeafSize cfg++-- | A care point with its index and which direction leads to correct classification+data CarePoint = CarePoint+    { cpIndex :: !Int+    , cpCorrectDir :: !Direction -- Which child classifies this point correctly+    }+    deriving (Eq, Show)++data Direction = GoLeft | GoRight+    deriving (Eq, Show)++{- | Identify care points: points where exactly one subtree classifies correctly++   For each point reaching the node:+   1. Compute what label the left subtree would predict+   2. Compute what label the right subtree would predict+   3. If exactly one matches the true label, it's a care point+   4. Record which direction leads to correct classification+-}+identifyCarePoints ::+    forall a.+    (Columnable a) =>+    T.Text ->+    DataFrame ->+    V.Vector Int ->+    Tree a -> -- Left subtree+    Tree a -> -- Right subtree+    [CarePoint]+identifyCarePoints target df indices leftTree rightTree =+    case interpret @a df (Col target) of+        Left _ -> []+        Right (TColumn col) ->+            case toVector @a col of+                Left _ -> []+                Right targetVals ->+                    V.toList $ V.mapMaybe (checkPoint targetVals) indices+  where+    checkPoint :: V.Vector a -> Int -> Maybe CarePoint+    checkPoint targetVals idx =+        let+            trueLabel = targetVals V.! idx+            leftPred = predictWithTree @a target df idx leftTree+            rightPred = predictWithTree @a target df idx rightTree+            leftCorrect = leftPred == trueLabel+            rightCorrect = rightPred == trueLabel+         in+            case (leftCorrect, rightCorrect) of+                (True, False) -> Just $ CarePoint idx GoLeft+                (False, True) -> Just $ CarePoint idx GoRight+                _ -> Nothing -- Don't-care point (both correct or both wrong)++-- | Predict the label for a single point using a fixed tree+predictWithTree ::+    forall a.+    (Columnable a) =>+    T.Text ->+    DataFrame ->+    Int -> -- Row index+    Tree a ->+    a+predictWithTree target df idx (Leaf v) = v+predictWithTree target df idx (Branch cond left right) =+    case interpret @Bool df cond of+        Left _ -> predictWithTree @a target df idx left -- Default to left on error+        Right (TColumn col) ->+            case toVector @Bool col of+                Left _ -> predictWithTree @a target df idx left+                Right boolVals ->+                    if boolVals V.! idx+                        then predictWithTree @a target df idx left+                        else predictWithTree @a target df idx right++countCarePointErrors :: Expr Bool -> DataFrame -> [CarePoint] -> Int+countCarePointErrors cond df carePoints =+    case interpret @Bool df cond of+        Left _ -> length carePoints+        Right (TColumn col) ->+            case toVector @Bool col of+                Left _ -> length carePoints+                Right boolVals ->+                    length $ filter (isMisclassified boolVals) carePoints+  where+    isMisclassified :: V.Vector Bool -> CarePoint -> Bool+    isMisclassified boolVals cp =+        let goesLeft = boolVals V.! cpIndex cp+            shouldGoLeft = cpCorrectDir cp == GoLeft+         in goesLeft /= shouldGoLeft++partitionIndices ::+    Expr Bool -> DataFrame -> V.Vector Int -> (V.Vector Int, V.Vector Int)+partitionIndices cond df indices =+    case interpret @Bool df cond of+        Left _ -> (indices, V.empty)+        Right (TColumn col) ->+            case toVector @Bool col of+                Left _ -> (indices, V.empty)+                Right boolVals ->+                    V.partition (boolVals V.!) indices++majorityValueFromIndices ::+    forall a.+    (Columnable a) =>+    T.Text ->+    DataFrame ->+    V.Vector Int ->+    a+majorityValueFromIndices target df indices =+    case interpret @a df (Col target) of+        Left e -> throw e+        Right (TColumn col) ->+            case toVector @a col of+                Left e -> throw e+                Right vals ->+                    let counts =+                            V.foldl'+                                (\acc i -> M.insertWith (+) (vals V.! i) 1 acc)+                                M.empty+                                indices+                     in if M.null counts+                            then error "Empty indices in majorityValueFromIndices"+                            else fst $ maximumBy (compare `on` snd) (M.toList counts)++computeTreeLoss ::+    forall a.+    (Columnable a) =>+    T.Text ->+    DataFrame ->+    V.Vector Int ->+    Tree a ->+    Double+computeTreeLoss target df indices tree+    | V.null indices = 0+    | otherwise =+        case interpret @a df (Col target) of+            Left _ -> 1.0+            Right (TColumn col) ->+                case toVector @a col of+                    Left _ -> 1.0+                    Right targetVals ->+                        let+                            n = V.length indices+                            errors =+                                V.length $+                                    V.filter+                                        (\i -> targetVals V.! i /= predictWithTree @a target df i tree)+                                        indices+                         in+                            fromIntegral errors / fromIntegral n++pruneDead :: Tree a -> Tree a+pruneDead (Leaf v) = Leaf v+pruneDead (Branch cond left right) =     let-        t = pruneTree trueBranch-        f = pruneTree falseBranch+        left' = pruneDead left+        right' = pruneDead right      in-        if t == f+        Branch cond left' right'++pruneExpr :: forall a. (Columnable a, Eq a) => Expr a -> Expr a+pruneExpr (If cond trueBranch falseBranch) =+    let t = pruneExpr trueBranch+        f = pruneExpr falseBranch+     in if t == f             then t             else case (t, f) of-                -- Nested simplification: `if C1 then (if C1 then X else Y) else Z`-                -- becomes:     if C1 then X else Z`-                -- Generalize this with hegg later.-                (If condInner tInner fInner, _) | cond == condInner -> If cond tInner f-                (_, If condInner tInner fInner) | cond == condInner -> If cond t fInner+                (If condInner tInner _, _) | cond == condInner -> If cond tInner f+                (_, If condInner _ fInner) | cond == condInner -> If cond t fInner                 _ -> If cond t f-pruneTree (UnaryOp name op e) = UnaryOp name op (pruneTree e)-pruneTree (BinaryOp name op l r) = BinaryOp name op (pruneTree l) (pruneTree r)-pruneTree e = e+pruneExpr (UnaryOp name op e) = UnaryOp name op (pruneExpr e)+pruneExpr (BinaryOp name op l r) = BinaryOp name op (pruneExpr l) (pruneExpr r)+pruneExpr e = e -type CondGen = TreeConfig -> DataFrame -> [Expr Bool]+buildGreedyTree ::+    forall a.+    (Columnable a) =>+    TreeConfig ->+    Int ->+    T.Text ->+    [Expr Bool] ->+    DataFrame ->+    Tree a+buildGreedyTree cfg depth target conds df+    | depth <= 0 || nRows df <= minSamplesSplit cfg =+        Leaf (majorityValue @a target df)+    | otherwise =+        case findBestGreedySplit @a cfg target conds df of+            Nothing -> Leaf (majorityValue @a target df)+            Just bestCond ->+                let (dfTrue, dfFalse) = partitionDataFrame bestCond df+                 in if nRows dfTrue < minLeafSize cfg || nRows dfFalse < minLeafSize cfg+                        then Leaf (majorityValue @a target df)+                        else+                            Branch+                                bestCond+                                (buildGreedyTree @a cfg (depth - 1) target conds dfTrue)+                                (buildGreedyTree @a cfg (depth - 1) target conds dfFalse) -numericConditions :: CondGen+findBestGreedySplit ::+    forall a.+    (Columnable a) =>+    TreeConfig -> T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool)+findBestGreedySplit cfg target conds df =+    let+        initialImpurity = calculateGini @a target df+        calculateComplexity c = complexityPenalty (synthConfig cfg) * fromIntegral (eSize c)++        evalGain :: Expr Bool -> (Double, Int)+        evalGain cond =+            let (t, f) = partitionDataFrame cond df+                n = fromIntegral @Int @Double (nRows df)+                weightT = fromIntegral @Int @Double (nRows t) / n+                weightF = fromIntegral @Int @Double (nRows f) / n+                newImpurity =+                    weightT * calculateGini @a target t+                        + weightF * calculateGini @a target f+             in ( (initialImpurity - newImpurity) - calculateComplexity cond+                , negate (eSize cond)+                )++        validConds =+            filter+                ( \c ->+                    let (t, f) = partitionDataFrame c df+                     in nRows t >= minLeafSize cfg && nRows f >= minLeafSize cfg+                )+                conds++        sortedConditions =+            map fst $+                take+                    (expressionPairs cfg)+                    ( filter+                        (\(c, v) -> ((> negate (calculateComplexity c)) . fst) v)+                        (sortBy (flip compare `on` snd) (map (\c -> (c, evalGain c)) validConds))+                    )+     in+        if null sortedConditions+            then Nothing+            else+                Just $+                    maximumBy+                        (compare `on` evalGain)+                        ( boolExprs+                            df+                            sortedConditions+                            sortedConditions+                            0+                            (boolExpansion (synthConfig cfg))+                        )++numericConditions :: TreeConfig -> DataFrame -> [Expr Bool] numericConditions = generateNumericConds -generateNumericConds ::-    TreeConfig -> DataFrame -> [Expr Bool]+generateNumericConds :: TreeConfig -> DataFrame -> [Expr Bool] generateNumericConds cfg df = do     expr <- numericExprsWithTerms (synthConfig cfg) df     let thresholds = map (\p -> percentile p expr df) (percentiles cfg)@@ -167,8 +562,7 @@         , expr .> F.lit threshold         ] -numericExprsWithTerms ::-    SynthConfig -> DataFrame -> [Expr Double]+numericExprsWithTerms :: SynthConfig -> DataFrame -> [Expr Double] numericExprsWithTerms cfg df =     concatMap (numericExprs cfg df [] 0) [0 .. maxExprDepth cfg] @@ -230,18 +624,14 @@         genConds :: T.Text -> [Expr Bool]         genConds colName = case unsafeGetColumn colName df of             (BoxedColumn (col :: V.Vector a)) ->-                let-                    percentiles = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99]-                 in-                    map (Col @a colName .==) percentiles+                let ps = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99]+                 in map (Col @a colName .==) ps             (OptionalColumn (col :: V.Vector (Maybe a))) -> case sFloating @a of                 STrue ->-                    let-                        doubleCol =+                    let doubleCol =                             VU.convert                                 (V.map fromJust (V.filter isJust (V.map (fmap (realToFrac @a @Double)) col)))-                     in-                        zipWith+                     in zipWith                             ($)                             [ (Col @(Maybe a) colName .==)                             , (Col @(Maybe a) colName .<=)@@ -249,21 +639,15 @@                             ]                             ( Lit Nothing                                 : map-                                    ( Lit-                                        . Just-                                        . realToFrac-                                        . (`percentile'` doubleCol)-                                    )+                                    (Lit . Just . realToFrac . (`percentile'` doubleCol))                                     (percentiles cfg)                             )                 SFalse -> case sIntegral @a of                     STrue ->-                        let-                            doubleCol =+                        let doubleCol =                                 VU.convert                                     (V.map fromJust (V.filter isJust (V.map (fmap (fromIntegral @a @Double)) col)))-                         in-                            zipWith+                         in zipWith                                 ($)                                 [ (Col @(Maybe a) colName .==)                                 , (Col @(Maybe a) colName .<=)@@ -271,18 +655,15 @@                                 ]                                 ( Lit Nothing                                     : map-                                        ( Lit-                                            . Just-                                            . round-                                            . (`percentile'` doubleCol)-                                        )+                                        (Lit . Just . round . (`percentile'` doubleCol))                                         (percentiles cfg)                                 )                     SFalse ->                         map                             ((Col @(Maybe a) colName .==) . Lit . (`percentileOrd'` col))                             [1, 25, 75, 99]-            (UnboxedColumn (col :: VU.Vector a)) -> []+            (UnboxedColumn (_ :: VU.Vector a)) -> []+         columnConds =             concatMap                 colConds@@ -297,12 +678,13 @@                 ]           where             colConds (!l, !r) = case (unsafeGetColumn l df, unsafeGetColumn r df) of-                (BoxedColumn (col1 :: V.Vector a), BoxedColumn (col2 :: V.Vector b)) -> case testEquality (typeRep @a) (typeRep @b) of-                    Nothing -> []-                    Just Refl -> [Col @a l .== Col @a r]-                (UnboxedColumn (col1 :: VU.Vector a), UnboxedColumn (col2 :: VU.Vector b)) -> []-                ( OptionalColumn (col1 :: V.Vector (Maybe a))-                    , OptionalColumn (col2 :: V.Vector (Maybe b))+                (BoxedColumn (col1 :: V.Vector a), BoxedColumn (_ :: V.Vector b)) ->+                    case testEquality (typeRep @a) (typeRep @b) of+                        Nothing -> []+                        Just Refl -> [Col @a l .== Col @a r]+                (UnboxedColumn (_ :: VU.Vector a), UnboxedColumn (_ :: VU.Vector b)) -> []+                ( OptionalColumn (_ :: V.Vector (Maybe a))+                    , OptionalColumn (_ :: V.Vector (Maybe b))                     ) -> case testEquality (typeRep @a) (typeRep @b) of                         Nothing -> []                         Just Refl -> case testEquality (typeRep @a) (typeRep @T.Text) of@@ -315,67 +697,7 @@ partitionDataFrame :: Expr Bool -> DataFrame -> (DataFrame, DataFrame) partitionDataFrame cond df = (filterWhere cond df, filterWhere (F.not cond) df) -findBestSplit ::-    forall a.-    (Columnable a) =>-    TreeConfig -> T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool)-findBestSplit cfg target conds df =-    let-        initialImpurity = calculateGini @a target df-        calculateComplexity c = complexityPenalty (synthConfig cfg) * fromIntegral (eSize c)-        evalGain :: Expr Bool -> (Double, Int)-        evalGain cond =-            let (t, f) = partitionDataFrame cond df-                n = fromIntegral @Int @Double (nRows df)-                weightT = fromIntegral @Int @Double (nRows t) / n-                weightF = fromIntegral @Int @Double (nRows f) / n-                newImpurity =-                    (weightT * calculateGini @a target t)-                        + (weightF * calculateGini @a target f)-             in ( (initialImpurity - newImpurity)-                    - calculateComplexity cond-                , negate (eSize cond)-                )--        validConds =-            filter-                ( \c ->-                    let-                        (t, f) = partitionDataFrame c df-                     in-                        nRows t >= minLeafSize cfg && nRows f >= minLeafSize cfg-                )-                conds-        sortedConditions =-            map fst $-                take-                    (expressionPairs cfg)-                    ( filter-                        ( \(c, v) ->-                            ((> negate (calculateComplexity c)) . fst)-                                v-                        )-                        (sortBy (flip compare `on` snd) (map (\c -> (c, evalGain c)) validConds))-                    )-     in-        if null sortedConditions-            then Nothing-            else-                Just $-                    maximumBy-                        (compare `on` evalGain)-                        ( boolExprs-                            df-                            sortedConditions-                            sortedConditions-                            0-                            (boolExpansion (synthConfig cfg))-                        )--calculateGini ::-    forall a.-    (Columnable a) =>-    T.Text -> DataFrame -> Double+calculateGini :: forall a. (Columnable a) => T.Text -> DataFrame -> Double calculateGini target df =     let n = fromIntegral $ nRows df         counts = getCounts @a target df@@ -383,20 +705,14 @@         probs = map (\c -> (fromIntegral c + 1) / (n + numClasses)) (M.elems counts)      in if n == 0 then 0 else 1 - sum (map (^ 2) probs) -majorityValue ::-    forall a.-    (Columnable a) =>-    T.Text -> DataFrame -> a+majorityValue :: forall a. (Columnable a) => T.Text -> DataFrame -> a majorityValue target df =     let counts = getCounts @a target df      in if M.null counts             then error "Empty DataFrame in leaf"             else fst $ maximumBy (compare `on` snd) (M.toList counts) -getCounts ::-    forall a.-    (Columnable a) =>-    T.Text -> DataFrame -> M.Map a Int+getCounts :: forall a. (Columnable a) => T.Text -> DataFrame -> M.Map a Int getCounts target df =     case interpret @a df (Col target) of         Left e -> throw e@@ -404,3 +720,42 @@             case toVector @a col of                 Left e -> throw e                 Right vals -> foldl' (\acc x -> M.insertWith (+) x 1 acc) M.empty (V.toList vals)++percentile :: Int -> Expr Double -> DataFrame -> Double+percentile p expr df =+    case interpret @Double df expr of+        Left _ -> 0+        Right (TColumn col) ->+            case toVector @Double col of+                Left _ -> 0+                Right vals ->+                    let sorted = V.fromList $ sort $ V.toList vals+                        n = V.length sorted+                        idx = min (n - 1) $ max 0 $ (p * n) `div` 100+                     in if n == 0 then 0 else sorted V.! idx++buildTree ::+    forall a.+    (Columnable a) =>+    TreeConfig ->+    Int ->+    T.Text ->+    [Expr Bool] ->+    DataFrame ->+    Expr a+buildTree cfg depth target conds df =+    let+        tree = buildGreedyTree @a cfg depth target conds df+        indices = V.enumFromN 0 (nRows df)+        optimized = taoOptimize @a cfg target conds df indices tree+     in+        pruneExpr (treeToExpr optimized)++findBestSplit ::+    forall a.+    (Columnable a) =>+    TreeConfig -> T.Text -> [Expr Bool] -> DataFrame -> Maybe (Expr Bool)+findBestSplit = findBestGreedySplit @a++pruneTree :: forall a. (Columnable a, Eq a) => Expr a -> Expr a+pruneTree = pruneExpr

@@ -167,6 +167,8 @@     -- ^ Character that separates column values.     , numColumns :: Maybe Int     -- ^ Number of columns to read.+    , missingIndicators :: [T.Text]+    -- ^ Values that should be read as `Nothing`.     }  shouldInferFromSample :: TypeSpec -> Bool@@ -190,6 +192,7 @@         , dateFormat = "%Y-%m-%d"         , columnSeparator = ','         , numColumns = Nothing+        , missingIndicators = []         }  {- | Read CSV file from path and load it into a dataframe.@@ -236,7 +239,8 @@ readSeparated :: ReadOptions -> FilePath -> IO DataFrame readSeparated !opts !path = do     let sep = columnSeparator opts-    csvData <- BL.readFile path+    let stripUtf8Bom bs = fromMaybe bs (BL.stripPrefix "\xEF\xBB\xBF" bs)+    csvData <- stripUtf8Bom <$> BL.readFile path     let decodeOpts = Csv.defaultDecodeOptions{Csv.decDelimiter = fromIntegral (ord sep)}     let stream = CsvStream.decodeWith decodeOpts Csv.NoHeader csvData @@ -263,7 +267,11 @@      (sampleRow, _) <- peekStream rowsToProcess     builderCols <- initializeColumns (V.toList sampleRow) opts-    processStream rowsToProcess builderCols (numColumns opts)+    processStream+        (missingIndicators opts)+        rowsToProcess+        builderCols+        (numColumns opts)      frozenCols <- V.fromList <$> mapM freezeBuilderColumn builderCols     let numRows = maybe 0 columnLength (frozenCols V.!? 0)@@ -278,6 +286,7 @@         if shouldInferFromSample (typeSpec opts)             then                 parseDefaults+                    (missingIndicators opts)                     (typeInferenceSampleSize (typeSpec opts))                     (safeRead opts)                     (dateFormat opts)@@ -305,17 +314,20 @@                     Nothing -> BuilderText <$> newPagedVector <*> pure validityRef  processStream ::+    [T.Text] ->     CsvStream.Records (V.Vector BL.ByteString) ->     [BuilderColumn] ->     Maybe Int ->     IO ()-processStream _ _ (Just 0) = return ()-processStream (Cons (Right row) rest) cols n = processRow row cols >> processStream rest cols (fmap (flip (-) 1) n)-processStream (Cons (Left err) _) _ _ = error ("CSV Parse Error: " ++ err)-processStream (Nil _ _) _ _ = return ()+processStream _ _ _ (Just 0) = return ()+processStream missing (Cons (Right row) rest) cols n =+    processRow missing row cols+        >> processStream missing rest cols (fmap (flip (-) 1) n)+processStream missing (Cons (Left err) _) _ _ = error ("CSV Parse Error: " ++ err)+processStream missing (Nil _ _) _ _ = return () -processRow :: V.Vector BL.ByteString -> [BuilderColumn] -> IO ()-processRow !vals !cols = V.zipWithM_ processValue vals (V.fromList cols)+processRow :: [T.Text] -> V.Vector BL.ByteString -> [BuilderColumn] -> IO ()+processRow missing !vals !cols = V.zipWithM_ processValue vals (V.fromList cols)   where     processValue !bs !col = do         let bs' = BL.toStrict bs@@ -328,7 +340,7 @@                 Nothing -> appendPagedUnboxedVector gv 0.0 >> appendPagedUnboxedVector valid 0             BuilderText gv valid -> do                 let !val = T.strip (TE.decodeUtf8Lenient bs')-                if isNullish val+                if isNullish val || val `elem` missing                     then appendPagedVector gv T.empty >> appendPagedUnboxedVector valid 0                     else appendPagedVector gv val >> appendPagedUnboxedVector valid 1 @@ -428,11 +440,9 @@                     ++ "the other columns at index "                     ++ show i     go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of-        Just e -> textRep : acc-          where-            textRep = case testEquality (typeRep @a) (typeRep @T.Text) of-                Just Refl -> fromMaybe "Nothing" e-                Nothing -> (T.pack . show) e+        Just e -> case testEquality (typeRep @a) (typeRep @T.Text) of+            Just Refl -> fromMaybe T.empty e : acc+            Nothing -> maybe T.empty (T.pack . show) e : acc         Nothing ->             error $                 "Column "

@@ -1,4 +1,7 @@+{-# LANGUAGE AllowAmbiguousTypes #-}+{-# LANGUAGE GADTs #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE StrictData #-} {-# LANGUAGE TypeApplications #-} @@ -13,9 +16,20 @@ import qualified Data.Map as M import Data.Maybe import qualified Data.Text as T+import Data.Typeable (Typeable)+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU import Data.Word import DataFrame.IO.Parquet.Binary import DataFrame.IO.Parquet.Types+import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.DataFrame (DataFrame, unsafeGetColumn)+import qualified DataFrame.Operations.Core as DI+import Type.Reflection (+    eqTypeRep,+    typeRep,+    (:~~:) (HRefl),+ )  data SchemaElement = SchemaElement     { elementName :: T.Text@@ -31,6 +45,39 @@     }     deriving (Show, Eq) +createParquetSchema :: DataFrame -> [SchemaElement]+createParquetSchema df = schemaDef : map toSchemaElement (DI.columnNames df)+  where+    -- The schema always contains an initial element+    -- indicating the group of fields.+    schemaDef =+        SchemaElement+            { elementName = "schema"+            , elementType = STOP+            , typeLength = 0+            , numChildren = fromIntegral (snd (DI.dimensions df))+            , fieldId = -1+            , repetitionType = UNKNOWN_REPETITION_TYPE+            , convertedType = 0+            , scale = 0+            , precision = 0+            , logicalType = LOGICAL_TYPE_UNKNOWN+            }+    toSchemaElement colName =+        let+            colType :: TType+            colType = case unsafeGetColumn colName df of+                (DI.BoxedColumn (col :: V.Vector a)) -> haskellToTType @a+                (DI.UnboxedColumn (col :: VU.Vector a)) -> haskellToTType @a+                (DI.OptionalColumn (col :: V.Vector (Maybe a))) -> haskellToTType @a+            lType =+                if DI.hasElemType @T.Text (unsafeGetColumn colName df)+                    || DI.hasElemType @(Maybe T.Text) (unsafeGetColumn colName df)+                    then STRING_TYPE+                    else LOGICAL_TYPE_UNKNOWN+         in+            SchemaElement colName colType 0 0 (-1) OPTIONAL 0 0 0 lType+ data KeyValue = KeyValue     { key :: String     , value :: String@@ -67,6 +114,29 @@     | STRUCT     | UUID     deriving (Show, Eq)++haskellToTType :: forall a. (Typeable a) => TType+haskellToTType+    | is @Bool = BOOL+    | is @Int8 = BYTE+    | is @Word8 = BYTE+    | is @Int16 = I16+    | is @Word16 = I16+    | is @Int32 = I32+    | is @Word32 = I32+    | is @Int64 = I64+    | is @Word64 = I64+    | is @Float = FLOAT+    | is @Double = DOUBLE+    | is @String = STRING+    | is @T.Text = STRING+    | is @BS.ByteString = STRING+    | otherwise = STOP+  where+    is :: forall x. (Typeable x) => Bool+    is = case eqTypeRep (typeRep @a) (typeRep @x) of+        Just HRefl -> True+        Nothing -> False  defaultMetadata :: FileMetadata defaultMetadata =

@@ -124,6 +124,7 @@                         then typeInferenceSampleSize (typeSpec opts)                         else 0              in parseFromExamples+                    (missingIndicators opts)                     n                     (safeRead opts)                     (dateFormat opts)

@@ -28,6 +28,7 @@  import Control.Exception (throw) import Control.Monad.ST (runST)+import Data.Kind (Type) import Data.Maybe import Data.Type.Equality (TestEquality (..)) import DataFrame.Errors@@ -139,20 +140,18 @@             Just Refl -> a == b     (==) _ _ = False +-- Generalised LEQ that does reflection.+generalLEQ ::+    forall a b. (Typeable a, Typeable b, Ord a, Ord b) => a -> b -> Bool+generalLEQ x y = case testEquality (typeRep @a) (typeRep @b) of+    Nothing -> False+    Just Refl -> x <= y+ instance Ord Column where     (<=) :: Column -> Column -> Bool-    (<=) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) =-        case testEquality (typeRep @t1) (typeRep @t2) of-            Nothing -> False-            Just Refl -> a <= b-    (<=) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) =-        case testEquality (typeRep @t1) (typeRep @t2) of-            Nothing -> False-            Just Refl -> a <= b-    (<=) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) =-        case testEquality (typeRep @t1) (typeRep @t2) of-            Nothing -> False-            Just Refl -> a <= b+    (<=) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) = generalLEQ a b+    (<=) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) = generalLEQ a b+    (<=) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) = generalLEQ a b     (<=) _ _ = False  {- | A class for converting a vector to a column of the appropriate type.@@ -239,63 +238,71 @@     [a] -> Column fromList = toColumnRep @(KindOf a) . VB.fromList +throwTypeMismatch ::+    forall (a :: Type) (b :: Type).+    (Typeable a, Typeable b) => Either DataFrameException Column+throwTypeMismatch =+    Left $+        TypeMismatchException+            MkTypeErrorContext+                { userType = Right (typeRep @b)+                , expectedType = Right (typeRep @a)+                , callingFunctionName = Just "mapColumn"+                , errorColumnName = Nothing+                }+ -- | An internal function to map a function over the values of a column. mapColumn ::     forall b c.-    ( Columnable b-    , Columnable c-    , UnboxIf c-    ) =>-    (b -> c) ->-    Column ->-    Either DataFrameException Column+    (Columnable b, Columnable c) =>+    (b -> c) -> Column -> Either DataFrameException Column mapColumn f = \case-    BoxedColumn (col :: VB.Vector a)-        | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Right (fromVector @c (VB.map f col))-        | otherwise ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @a)-                        , callingFunctionName = Just "mapColumn"-                        , errorColumnName = Nothing-                        }-                    )-    OptionalColumn (col :: VB.Vector a)-        | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Right (fromVector @c (VB.map f col))-        | otherwise ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @a)-                        , callingFunctionName = Just "mapColumn"-                        , errorColumnName = Nothing-                        }-                    )-    UnboxedColumn (col :: VU.Vector a)-        | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            Right $ case sUnbox @c of-                STrue -> UnboxedColumn (VU.map f col)-                SFalse -> fromVector @c (VB.generate (VU.length col) (f . VU.unsafeIndex col))-        | otherwise ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @a)-                        , callingFunctionName = Just "mapColumn"-                        , errorColumnName = Nothing-                        }-                    )+    BoxedColumn (col :: VB.Vector a) -> run col+    OptionalColumn (col :: VB.Vector a) -> run col+    UnboxedColumn (col :: VU.Vector a) -> runUnboxed col+  where+    run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column+    run col = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right (fromVector @c (VB.map f col))+        Nothing -> throwTypeMismatch @a @b++    runUnboxed ::+        forall a.+        (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column+    runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right $ case sUnbox @c of+            STrue -> UnboxedColumn (VU.map f col)+            SFalse -> fromVector @c (VB.generate (VU.length col) (f . VU.unsafeIndex col))+        Nothing -> throwTypeMismatch @a @b {-# SPECIALIZE mapColumn ::     (Double -> Double) -> Column -> Either DataFrameException Column     #-} {-# INLINEABLE mapColumn #-} +-- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column.+imapColumn ::+    forall b c.+    (Columnable b, Columnable c) =>+    (Int -> b -> c) -> Column -> Either DataFrameException Column+imapColumn f = \case+    BoxedColumn (col :: VB.Vector a) -> run col+    OptionalColumn (col :: VB.Vector a) -> run col+    UnboxedColumn (col :: VU.Vector a) -> runUnboxed col+  where+    run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column+    run col = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right (fromVector @c (VB.imap f col))+        Nothing -> throwTypeMismatch @a @b++    runUnboxed ::+        forall a.+        (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column+    runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right $ case sUnbox @c of+            STrue -> UnboxedColumn (VU.imap f col)+            SFalse -> BoxedColumn (VB.imap f (VG.convert col))+        Nothing -> throwTypeMismatch @a @b+ -- | O(1) Gets the number of elements in the column. columnLength :: Column -> Int columnLength (BoxedColumn xs) = VG.length xs@@ -373,158 +380,46 @@     (a -> Bool) ->     Column ->     Either DataFrameException (VU.Vector Int)-findIndices pred (BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of-    Just Refl -> pure $ VG.convert (VG.findIndices pred column)-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "findIndices"-                    , errorColumnName = Nothing-                    }-                )-findIndices pred (UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of-    Just Refl -> pure $ VG.findIndices pred column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "findIndices"-                    , errorColumnName = Nothing-                    }-                )-findIndices pred (OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of-    Just Refl -> pure $ VG.convert (VG.findIndices pred column)-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "findIndices"-                    , errorColumnName = Nothing-                    }-                )---- | An internal function that returns a vector of how indexes change after a column is sorted.-sortedIndexes :: Bool -> Column -> VU.Vector Int-sortedIndexes asc (BoxedColumn column) = runST $ do-    withIndexes <- VG.thaw $ VG.indexed column-    VA.sortBy-        (\(a, b) (a', b') -> (if asc then compare else flip compare) b b')-        withIndexes-    sorted <- VG.unsafeFreeze withIndexes-    return $ VU.generate (VG.length column) (\i -> fst (sorted VG.! i))-sortedIndexes asc (UnboxedColumn column) = runST $ do-    withIndexes <- VG.thaw $ VG.indexed column-    VA.sortBy-        (\(a, b) (a', b') -> (if asc then compare else flip compare) b b')-        withIndexes-    sorted <- VG.unsafeFreeze withIndexes-    return $ VU.generate (VG.length column) (\i -> fst (sorted VG.! i))-sortedIndexes asc (OptionalColumn column) = runST $ do-    withIndexes <- VG.thaw $ VG.indexed column-    VA.sortBy-        (\(a, b) (a', b') -> (if asc then compare else flip compare) b b')-        withIndexes-    sorted <- VG.unsafeFreeze withIndexes-    return $ VU.generate (VG.length column) (\i -> fst (sorted VG.! i))-{-# INLINE sortedIndexes #-}---- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column.-imapColumn ::-    forall b c.-    (Columnable b, Columnable c) =>-    (Int -> b -> c) -> Column -> Either DataFrameException Column-imapColumn f = \case-    BoxedColumn (col :: VB.Vector a)-        | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            pure (fromVector @c (VB.imap f col))-        | otherwise ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @a)-                        , callingFunctionName = Just "imapColumn"-                        , errorColumnName = Nothing-                        }-                    )-    UnboxedColumn (col :: VU.Vector a)-        | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            pure $-                case sUnbox @c of-                    STrue -> UnboxedColumn (VU.imap f col)-                    SFalse -> fromVector @c (VB.imap f (VB.convert col))-        | otherwise ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @a)-                        , callingFunctionName = Just "imapColumn"-                        , errorColumnName = Nothing-                        }-                    )-    OptionalColumn (col :: VB.Vector a)-        | Just Refl <- testEquality (typeRep @a) (typeRep @b) ->-            pure (fromVector @c (VB.imap f col))-        | otherwise ->+findIndices pred = \case+    BoxedColumn (v :: VB.Vector b) -> run v VG.convert+    OptionalColumn (v :: VB.Vector b) -> run v VG.convert+    UnboxedColumn (v :: VU.Vector b) -> run v id+  where+    run ::+        forall b v.+        (Typeable b, VG.Vector v b, VG.Vector v Int) =>+        v b ->+        (v Int -> VU.Vector Int) ->+        Either DataFrameException (VU.Vector Int)+    run column finalize = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right . finalize $ VG.findIndices pred column+        Nothing ->             Left $                 TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @a)-                        , callingFunctionName = Just "imapColumn"+                    MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Right (typeRep @b)+                        , callingFunctionName = Just "findIndices"                         , errorColumnName = Nothing                         }-                    ) --- | Filter column with index.-ifilterColumn ::-    forall a.-    (Columnable a) =>-    (Int -> a -> Bool) -> Column -> Either DataFrameException Column-ifilterColumn f c@(BoxedColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of-    Just Refl -> pure $ BoxedColumn $ VG.ifilter f column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "ifilterColumn"-                    , errorColumnName = Nothing-                    }-                )-ifilterColumn f c@(UnboxedColumn (column :: VU.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of-    Just Refl -> pure $ UnboxedColumn $ VG.ifilter f column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "ifilterColumn"-                    , errorColumnName = Nothing-                    }-                )-ifilterColumn f c@(OptionalColumn (column :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of-    Just Refl -> pure $ OptionalColumn $ VG.ifilter f column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "ifilterColumn"-                    , errorColumnName = Nothing-                    }-                )+-- | An internal function that returns a vector of how indexes change after a column is sorted.+sortedIndexes :: Bool -> Column -> VU.Vector Int+sortedIndexes asc = \case+    BoxedColumn column -> sortWorker column+    UnboxedColumn column -> sortWorker column+    OptionalColumn column -> sortWorker column+  where+    sortWorker ::+        (VG.Vector v a, Ord a, VG.Vector v (Int, a), VG.Vector v Int) =>+        v a -> VU.Vector Int+    sortWorker column = runST $ do+        withIndexes <- VG.thaw $ VG.indexed column+        let cmp = if asc then compare else flip compare+        VA.sortBy (\(_, b) (_, b') -> cmp b b') withIndexes+        sorted <- VG.unsafeFreeze withIndexes+        return $ VG.convert $ VG.map fst sorted+{-# INLINE sortedIndexes #-}  -- | Fold (right) column with index. ifoldrColumn ::@@ -568,48 +463,6 @@                     }                 ) --- | Fold (left) column with index.-ifoldlColumn ::-    forall a b.-    (Columnable a, Columnable b) =>-    (b -> Int -> a -> b) -> b -> Column -> Either DataFrameException b-ifoldlColumn f acc c@(BoxedColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of-    Just Refl -> pure $ VG.ifoldl' f acc column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @d)-                    , callingFunctionName = Just "ifoldlColumn"-                    , errorColumnName = Nothing-                    }-                )-ifoldlColumn f acc c@(OptionalColumn (column :: VB.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of-    Just Refl -> pure $ VG.ifoldl' f acc column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @d)-                    , callingFunctionName = Just "ifoldlColumn"-                    , errorColumnName = Nothing-                    }-                )-ifoldlColumn f acc c@(UnboxedColumn (column :: VU.Vector d)) = case testEquality (typeRep @a) (typeRep @d) of-    Just Refl -> pure $ VG.ifoldl' f acc column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @d)-                    , callingFunctionName = Just "ifoldlColumn"-                    , errorColumnName = Nothing-                    }-                )- foldlColumn ::     forall a b.     (Columnable a, Columnable b) =>@@ -738,55 +591,6 @@                     , errorColumnName = Nothing                     }                 )---- | Generic reduce function for all Column types.-reduceColumn ::-    forall a b.-    (Columnable a, Columnable b) =>-    (a -> b) -> Column -> Either DataFrameException b-{-# SPECIALIZE reduceColumn ::-    (VU.Vector (Double, Double) -> Double) ->-    Column ->-    Either DataFrameException Double-    , (VU.Vector Double -> Double) -> Column -> Either DataFrameException Double-    #-}-reduceColumn f (BoxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of-    Just Refl -> pure $ f column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "reduceColumn"-                    , errorColumnName = Nothing-                    }-                )-reduceColumn f (UnboxedColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of-    Just Refl -> pure $ f column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "reduceColumn"-                    , errorColumnName = Nothing-                    }-                )-reduceColumn f (OptionalColumn (column :: c)) = case testEquality (typeRep @c) (typeRep @a) of-    Just Refl -> pure $ f column-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @b)-                    , callingFunctionName = Just "reduceColumn"-                    , errorColumnName = Nothing-                    }-                )-{-# INLINE reduceColumn #-}  -- | An internal, column version of zip. zipColumns :: Column -> Column -> Column

@@ -383,11 +383,9 @@                     ++ "the other columns at index "                     ++ show i     go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of-        Just e -> textRep : acc-          where-            textRep = case testEquality (typeRep @a) (typeRep @T.Text) of-                Just Refl -> fromMaybe "Nothing" e-                Nothing -> (T.pack . show) e+        Just e -> case testEquality (typeRep @a) (typeRep @T.Text) of+            Just Refl -> fromMaybe T.empty e : acc+            Nothing -> maybe T.empty (T.pack . show) e : acc         Nothing ->             error $                 "Column "

@@ -14,6 +14,7 @@ import DataFrame.Internal.Expression (Expr (..))  import qualified Data.Text as T+import System.Random  -- A re-implementation of the state monad. -- `mtl` might be too heavy a dependency just to get@@ -60,6 +61,12 @@  filterWhereM :: Expr Bool -> FrameM () filterWhereM p = modifyM (D.filterWhere p)++sampleM :: (RandomGen g) => g -> Double -> FrameM ()+sampleM pureGen p = modifyM (D.sample pureGen p)++takeM :: Int -> FrameM ()+takeM n = modifyM (D.take n)  filterJustM :: (Columnable a) => Expr (Maybe a) -> FrameM (Expr a) filterJustM (Col name) = FrameM $ \df ->

@@ -864,7 +864,8 @@ This may occur if the column contains non-numeric data or values outside the 'Int' range. -} columnAsIntVector ::-    Expr Int -> DataFrame -> Either DataFrameException (VU.Vector Int)+    (Columnable a, Num a) =>+    Expr a -> DataFrame -> Either DataFrameException (VU.Vector Int) columnAsIntVector (Col name) df = case getColumn name df of     Just col -> toIntVector col     Nothing ->@@ -880,7 +881,8 @@ This may occur if the column contains non-numeric data. -} columnAsDoubleVector ::-    Expr Double -> DataFrame -> Either DataFrameException (VU.Vector Double)+    (Columnable a, Num a) =>+    Expr a -> DataFrame -> Either DataFrameException (VU.Vector Double) columnAsDoubleVector (Col name) df = case getColumn name df of     Just col -> toDoubleVector col     Nothing ->@@ -896,7 +898,8 @@ This may occur if the column contains non-numeric data. -} columnAsFloatVector ::-    Expr Float -> DataFrame -> Either DataFrameException (VU.Vector Float)+    (Columnable a, Num a) =>+    Expr a -> DataFrame -> Either DataFrameException (VU.Vector Float) columnAsFloatVector (Col name) df = case getColumn name df of     Just col -> toFloatVector col     Nothing ->

@@ -18,11 +18,23 @@  import Control.Exception (throw) import Data.Function ((&))-import Data.Maybe (fromJust, fromMaybe, isJust, isNothing)+import Data.Maybe (+    fromJust,+    fromMaybe,+    isJust,+    isNothing,+ ) import Data.Type.Equality (TestEquality (..))-import DataFrame.Errors (DataFrameException (..), TypeErrorContext (..))+import DataFrame.Errors (+    DataFrameException (..),+    TypeErrorContext (..),+ ) import DataFrame.Internal.Column-import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn)+import DataFrame.Internal.DataFrame (+    DataFrame (..),+    empty,+    getColumn,+ ) import DataFrame.Internal.Expression import DataFrame.Internal.Interpreter import DataFrame.Operations.Core

@@ -22,28 +22,35 @@  type DateFormat = String -parseDefaults :: Int -> Bool -> DateFormat -> DataFrame -> DataFrame-parseDefaults n safeRead dateFormat df = df{columns = V.map (parseDefault n safeRead dateFormat) (columns df)}+parseDefaults :: [T.Text] -> Int -> Bool -> DateFormat -> DataFrame -> DataFrame+parseDefaults missing n safeRead dateFormat df = df{columns = V.map (parseDefault missing n safeRead dateFormat) (columns df)} -parseDefault :: Int -> Bool -> DateFormat -> Column -> Column-parseDefault n safeRead dateFormat (BoxedColumn (c :: V.Vector a)) =+parseDefault :: [T.Text] -> Int -> Bool -> DateFormat -> Column -> Column+parseDefault missing n safeRead dateFormat (BoxedColumn (c :: V.Vector a)) =     case (typeRep @a) `testEquality` (typeRep @T.Text) of         Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of-            Just Refl -> parseFromExamples n safeRead dateFormat (V.map T.pack c)+            Just Refl -> parseFromExamples missing n safeRead dateFormat (V.map T.pack c)             Nothing -> BoxedColumn c-        Just Refl -> parseFromExamples n safeRead dateFormat c-parseDefault n safeRead dateFormat (OptionalColumn (c :: V.Vector (Maybe a))) =+        Just Refl -> parseFromExamples missing n safeRead dateFormat c+parseDefault missing n safeRead dateFormat (OptionalColumn (c :: V.Vector (Maybe a))) =     case (typeRep @a) `testEquality` (typeRep @T.Text) of         Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of-            Just Refl -> parseFromExamples n safeRead dateFormat (V.map (T.pack . fromMaybe "") c)+            Just Refl ->+                parseFromExamples+                    missing+                    n+                    safeRead+                    dateFormat+                    (V.map (T.pack . fromMaybe "") c)             Nothing -> BoxedColumn c-        Just Refl -> parseFromExamples n safeRead dateFormat (V.map (fromMaybe "") c)-parseDefault _ _ _ column = column+        Just Refl -> parseFromExamples missing n safeRead dateFormat (V.map (fromMaybe "") c)+parseDefault _ _ _ _ column = column -parseFromExamples :: Int -> Bool -> DateFormat -> V.Vector T.Text -> Column-parseFromExamples n safeRead dateFormat cols =+parseFromExamples ::+    [T.Text] -> Int -> Bool -> DateFormat -> V.Vector T.Text -> Column+parseFromExamples missing n safeRead dateFormat cols =     let-        converter = if safeRead then convertNullish else convertOnlyEmpty+        converter = if safeRead then convertNullish missing else convertOnlyEmpty         examples = V.map converter (V.take n cols)         asMaybeText = V.map converter cols      in@@ -123,8 +130,8 @@     parsableAsDouble = vecSameConstructor asMaybeText asMaybeDouble     parsableAsDate = vecSameConstructor asMaybeText asMaybeDate -convertNullish :: T.Text -> Maybe T.Text-convertNullish v = if isNullish v then Nothing else Just v+convertNullish :: [T.Text] -> T.Text -> Maybe T.Text+convertNullish missing v = if isNullish v || v `elem` missing then Nothing else Just v  convertOnlyEmpty :: T.Text -> Maybe T.Text convertOnlyEmpty v = if v == "" then Nothing else Just v

@@ -266,7 +266,9 @@ percentiles df =     let         doubleColumns =-            map (either throw id . ((`columnAsDoubleVector` df) . Col)) (D.columnNames df)+            map+                (either throw id . ((`columnAsDoubleVector` df) . Col @Double))+                (D.columnNames df)      in         concatMap             (\c -> map (Lit . roundTo2SigDigits . (`percentile'` c)) [1, 25, 75, 99])

@@ -0,0 +1,42 @@+{-# LANGUAGE GADTs #-}+{-# LANGUAGE TypeApplications #-}++module GenDataFrame where++import qualified Data.Map as M+import qualified Data.Text as T+import qualified Data.Vector as V+import qualified Data.Vector.Unboxed as VU+import DataFrame.Internal.Column+import DataFrame.Internal.DataFrame+import Test.QuickCheck++genColumn :: Int -> Gen Column+genColumn len =+    oneof+        [ BoxedColumn . V.fromList <$> vectorOf len (arbitrary @Int)+        , UnboxedColumn . VU.fromList <$> vectorOf len (arbitrary @Double)+        , OptionalColumn . V.fromList <$> vectorOf len (arbitrary @(Maybe Int))+        ]++genDataFrame :: Gen DataFrame+genDataFrame = do+    numRows <- choose (0, 100)+    numCols <- choose (0, 10)+    colNames <- V.fromList <$> vectorOf numCols genUniqueColName+    cols <- V.fromList <$> vectorOf numCols (genColumn numRows)+    let indices = M.fromList $ zip (V.toList colNames) [0 ..]+    pure $+        DataFrame+            { columns = cols+            , columnIndices = indices+            , dataframeDimensions = (numRows, numCols)+            , derivingExpressions = M.empty+            }++genUniqueColName :: Gen T.Text+genUniqueColName = T.pack <$> listOf1 (elements ['a' .. 'z'])++instance Arbitrary DataFrame where+    arbitrary = genDataFrame+    shrink df = []

@@ -1,4 +1,5 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-}  module Main where@@ -12,7 +13,9 @@ import qualified System.Exit as Exit  import Data.Time+import GenDataFrame () import Test.HUnit+import Test.QuickCheck  import qualified Functions import qualified Operations.Aggregations@@ -27,6 +30,7 @@ import qualified Operations.ReadCsv import qualified Operations.Sort import qualified Operations.Statistics+import qualified Operations.Subset import qualified Operations.Take import qualified Parquet @@ -59,7 +63,7 @@         beforeParse :: [T.Text]         beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Bools without missing values as UnboxedColumn of Bools"@@ -74,7 +78,7 @@         beforeParse :: [T.Text]         beforeParse = T.pack . show <$> [1 .. 50]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints without missing values as UnboxedColumn of Ints"@@ -91,7 +95,7 @@             T.pack . show                 <$> [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Doubles without missing values as UnboxedColumn of Doubles"@@ -138,7 +142,7 @@             , "2020-02-26"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Dates without missing values as BoxedColumn of Days"@@ -223,7 +227,7 @@             , "Meowth, that's right!"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Text without missing values as BoxedColumn of Text"@@ -239,7 +243,7 @@         beforeParse :: [T.Text]         beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses mixture of Bools and Ints as Text"@@ -466,7 +470,7 @@             , "12.22222235049451"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles"@@ -561,7 +565,7 @@             , "2020-02-20"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints and Dates as BoxedColumn of Texts"@@ -700,7 +704,7 @@             , "12.22222235049451"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Texts and Doubles as BoxedColumn of Texts"@@ -753,7 +757,7 @@             , "Meowth"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Dates and Texts as BoxedColumn of Texts"@@ -770,7 +774,8 @@         beforeParse :: [T.Text]         beforeParse = replicate 10 "true" ++ replicate 10 "false"         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Bools without missing values as UnboxedColumn of Bools, when safeRead is off"@@ -887,7 +892,8 @@             , "50"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints without missing values as UnboxedColumn of Ints, when safeRead is off"@@ -1014,7 +1020,8 @@             , "12.22222235049451"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Doubles without missing values as UnboxedColumn of Doubles, when safeRead is off"@@ -1061,7 +1068,8 @@             , "2020-02-26"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Dates without missing values as BoxedColumn of Days"@@ -1146,7 +1154,8 @@             , "Meowth, that's right!"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Text without missing values as BoxedColumn of Text"@@ -1161,7 +1170,8 @@         beforeParse :: [T.Text]         beforeParse = replicate 10 "" ++ replicate 10 "true" ++ replicate 10 "false"         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Bools and empty Strings as OptionalColumn of Bools, when safeRead is off"@@ -1288,7 +1298,8 @@             , "50"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is off"@@ -1425,7 +1436,8 @@             , "12.22222235049451"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off"@@ -1482,7 +1494,8 @@             , ""             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead off"@@ -1581,7 +1594,8 @@             , "Meowth, that's right!"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off"@@ -1596,7 +1610,8 @@         beforeParse :: [T.Text]         beforeParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False"         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Bools with nullish values as BoxedColumn of Texts, when safeRead is off"@@ -1723,7 +1738,8 @@             , "50"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints with nullish values as BoxedColumn of Texts, when safeRead is off"@@ -1860,7 +1876,8 @@             , "12.22222235049451"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off"@@ -1999,7 +2016,8 @@             , ""             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Texts, when safeRead is off"@@ -2104,7 +2122,8 @@             , "N/A"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 10 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off"@@ -2120,7 +2139,7 @@         beforeParse :: [T.Text]         beforeParse = replicate 10 "" ++ replicate 10 "true"         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Bools and empty strings as OptionalColumn of Bools, when safeRead is on"@@ -2247,7 +2266,7 @@             , "50"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is on"@@ -2384,7 +2403,7 @@             , "12.22222235049451"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is on"@@ -2441,7 +2460,7 @@             , ""             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead on"@@ -2540,7 +2559,7 @@             , "Meowth, that's right!"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead on"@@ -2667,7 +2686,7 @@             , "50"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints with nullish values as OptionalColumn of Ints, when safeRead is on"@@ -2804,7 +2823,7 @@             , "12.03"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off"@@ -2943,7 +2962,7 @@             , ""             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Ints, when safeRead is on"@@ -3040,7 +3059,7 @@             , "3.14"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints and Doubles with nullish values AND empty strings as OptionalColumn of Doubles, when safeRead is on"@@ -3145,7 +3164,7 @@             , "N/A"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off"@@ -3161,7 +3180,7 @@         beforeParse :: [T.Text]         beforeParse = "false" : replicate 50 "true"         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example"@@ -3176,7 +3195,7 @@         beforeParse :: [T.Text]         beforeParse = "false" : replicate 50 "true"         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example"@@ -3293,7 +3312,7 @@             , "50"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints without missing values as UnboxedColumn of Ints with only one example"@@ -3410,7 +3429,7 @@             , "50"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 25 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 25 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints without missing values as UnboxedColumn of Ints with some examples"@@ -3527,7 +3546,7 @@             , "50"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 49 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Ints without missing values as UnboxedColumn of Ints with many examples"@@ -3574,7 +3593,7 @@             , "2020-02-26"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Dates without missing values as BoxedColumn of Days with only one example"@@ -3621,7 +3640,7 @@             , "2020-02-26"             ]         expected = DI.BoxedColumn $ V.fromList afterParse-        actual = D.parseDefault 15 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 15 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses Dates without missing values as BoxedColumn of Days with many examples"@@ -3848,7 +3867,7 @@             , "12.22222235049451"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 1 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example"@@ -4075,7 +4094,7 @@             , "12.22222235049451"             ]         expected = DI.UnboxedColumn $ VU.fromList afterParse-        actual = D.parseDefault 50 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 50 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example"@@ -4322,7 +4341,7 @@             , "12.22222235049451"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example"@@ -4570,7 +4589,8 @@             , "12.22222235049451"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example"@@ -4738,7 +4758,7 @@             , "12.22222235049451"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 1 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with just one example, when safeRead is off"@@ -4906,7 +4926,8 @@             , "12.22222235049451"             ]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual =+            D.parseDefault [] 30 False "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             ( assertEqual                 "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with many examples, when safeRead is off"@@ -4923,7 +4944,7 @@         beforeParse :: [T.Text]         beforeParse = replicate 100 "NaN" ++ ["100000"]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4934,7 +4955,7 @@         beforeParse :: [T.Text]         beforeParse = replicate 100 "NaN" ++ ["3.14"]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4945,7 +4966,7 @@         beforeParse :: [T.Text]         beforeParse = replicate 100 "NaN" ++ ["2024-12-25"]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4956,7 +4977,7 @@         beforeParse :: [T.Text]         beforeParse = replicate 100 "NaN" ++ ["2024-12-25", "2024-12-w6"]         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -4967,7 +4988,7 @@         beforeParse :: [T.Text]         beforeParse = replicate 100 "NaN"         expected = DI.OptionalColumn $ V.fromList afterParse-        actual = D.parseDefault 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse+        actual = D.parseDefault [] 10 True "%Y-%m-%d" $ DI.fromVector $ V.fromList beforeParse      in TestCase             (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) @@ -5105,9 +5126,18 @@             ++ Parquet.tests             ++ parseTests +isSuccessful :: Result -> Bool+isSuccessful (Success{..}) = True+isSuccessful _ = False+ main :: IO () main = do     result <- runTestTT tests-    if failures result > 0 || errors result > 0+    -- Property tests+    propRes <-+        mapM+            (quickCheckWithResult stdArgs)+            Operations.Subset.tests+    if failures result > 0 || errors result > 0 || not (all isSuccessful propRes)         then Exit.exitFailure         else Exit.exitSuccess

@@ -0,0 +1,95 @@+{-# LANGUAGE GADTs #-}++module Operations.Subset where++import DataFrame.Internal.DataFrame+import qualified DataFrame.Operations.Core as D+import qualified DataFrame.Operations.Subset as D+import System.Random++prop_dropZero :: DataFrame -> Bool+prop_dropZero df = D.drop 0 df == df++prop_takeZero :: DataFrame -> Bool+prop_takeZero df = fst (dataframeDimensions (D.take 0 df)) == 0++prop_takeAll :: DataFrame -> Bool+prop_takeAll df =+    let n = fst (dataframeDimensions df)+     in D.take n df == df++prop_dropAll :: DataFrame -> Bool+prop_dropAll df =+    let n = fst (dataframeDimensions df)+     in fst (dataframeDimensions (D.drop n df)) == 0++prop_takeLastZero :: DataFrame -> Bool+prop_takeLastZero df = fst (dataframeDimensions (D.takeLast 0 df)) == 0++prop_dropLastZero :: DataFrame -> Bool+prop_dropLastZero df = D.dropLast 0 df == df++prop_takeLastAll :: DataFrame -> Bool+prop_takeLastAll df =+    let n = fst (dataframeDimensions df)+     in D.takeLast n df == df++prop_dropLastAll :: DataFrame -> Bool+prop_dropLastAll df =+    let n = fst (dataframeDimensions df)+     in fst (dataframeDimensions (D.dropLast n df)) == 0++prop_rangeEmpty :: DataFrame -> Bool+prop_rangeEmpty df =+    fst (dataframeDimensions (D.range (5, 5) df)) == 0++prop_rangeFull :: DataFrame -> Bool+prop_rangeFull df =+    let rows = fst (dataframeDimensions df)+     in D.range (0, rows) df == df++prop_selectAll :: DataFrame -> Bool+prop_selectAll df = D.select (D.columnNames df) df == df++prop_selectEmpty :: DataFrame -> Bool+prop_selectEmpty df =+    let result = D.select [] df+     in dataframeDimensions result == (0, 0)++prop_excludeEmpty :: DataFrame -> Bool+prop_excludeEmpty df = D.exclude [] df == df++prop_excludeAll :: DataFrame -> Bool+prop_excludeAll df =+    let result = D.exclude (D.columnNames df) df+     in snd (dataframeDimensions result) == 0++prop_cubePreservesSmall :: DataFrame -> Bool+prop_cubePreservesSmall df =+    let (rows, cols) = dataframeDimensions df+     in D.cube (rows + 100, cols + 100) df == df++prop_sampleEmptyApprox :: DataFrame -> Bool+prop_sampleEmptyApprox df =+    let gen = mkStdGen 42+        sampled = D.sample gen 0.0 df+     in fst (dataframeDimensions sampled) == 0++tests =+    [ prop_dropZero+    , prop_takeZero+    , prop_takeAll+    , prop_dropAll+    , prop_takeLastZero+    , prop_dropLastZero+    , prop_takeLastAll+    , prop_dropLastAll+    , prop_rangeEmpty+    , prop_rangeFull+    , prop_selectAll+    , prop_selectEmpty+    , prop_excludeEmpty+    , prop_excludeAll+    , prop_cubePreservesSmall+    , prop_sampleEmptyApprox+    ]