diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Revision history for dataframe +## 3.4.0.0 + +* `impute` on a non-nullable expression is now fails and throws when given a column with the wrong type. +* `fromRows` throws on a cell whose type differs from its column's, and on a row too short to reach a column. +* `F.add`, `F.sub`, `F.mult`, `F.divide`, `F.prettyPrint` are now visible. +* `F.isNull` / `F.isNotNull` as aliases for `isNothing` / `isJust`. +* `DataFrame.Monad` now implements `selectM`, `excludeM` and `sortByM`. + ## 3.3.0.0 ### Breaking changes diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe -version: 3.3.0.0 +version: 3.4.0.0 synopsis: A fast, safe, and intuitive DataFrame library. description: A fast, safe, and intuitive DataFrame library for exploratory data analysis. @@ -128,13 +128,13 @@ DataFrame.Typed.Record, DataFrame.Typed.Generic build-depends: base >= 4 && <5, - dataframe-core >= 2.2 && < 2.3, + dataframe-core >= 2.3 && < 2.4, dataframe-json >= 1.2.0.1 && < 1.3, dataframe-expr-serializer >= 1.2.0.1 && < 1.3, - dataframe-operations >= 2.2 && < 2.3, + dataframe-operations >= 2.3 && < 2.4, dataframe-parsing >= 2.2 && < 2.3, dataframe-viz >= 1.3 && < 1.4, - dataframe-learn >= 2.2 && < 2.3 + dataframe-learn >= 2.3 && < 2.4 if !flag(no-csv) reexported-modules: DataFrame.IO.CSV, @@ -203,12 +203,12 @@ buildable: False build-depends: base >= 4 && < 5, - dataframe-core >= 2.2 && < 2.3, + dataframe-core >= 2.3 && < 2.4, dataframe-expr-serializer >= 1.2.0.1 && < 1.3, dataframe-csv >= 2.3 && < 2.4, dataframe-json >= 1.2.0.1 && < 1.3, dataframe-lazy >= 2.3 && < 2.4, - dataframe-operations >= 2.2 && < 2.3, + dataframe-operations >= 2.3 && < 2.4, dataframe-parquet >= 1.5 && < 1.6, dataframe-parsing >= 2.2 && < 2.3, text >= 2.1 && < 3, @@ -223,8 +223,8 @@ import: warnings main-is: Benchmark.hs build-depends: base >= 4 && < 5, - dataframe >= 3.3 && < 3.4, - dataframe-operations >= 2.2 && < 2.3, + dataframe >= 3.4 && < 3.5, + dataframe-operations >= 2.3 && < 2.4, random >= 1 && < 2, time >= 1.12 && < 2, vector >= 0.13 && < 0.15, @@ -236,10 +236,10 @@ import: warnings main-is: Synthesis.hs build-depends: base >= 4 && < 5, - dataframe >= 3.3 && < 3.4, - dataframe-core >= 2.2 && < 2.3, - dataframe-learn >= 2.2 && < 2.3, - dataframe-operations >= 2.2 && < 2.3, + dataframe >= 3.4 && < 3.5, + dataframe-core >= 2.3 && < 2.4, + dataframe-learn >= 2.3 && < 2.4, + dataframe-operations >= 2.3 && < 2.4, random >= 1 && < 2, text >= 2.1 && < 3 hs-source-dirs: app @@ -270,8 +270,8 @@ build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, containers >= 0.6.7 && < 0.10, - dataframe >= 3.3 && < 3.4, - dataframe-core >= 2.2 && < 2.3, + dataframe >= 3.4 && < 3.5, + dataframe-core >= 2.3 && < 2.4, dataframe-lazy >= 2.3 && < 2.4, dataframe-parsing >= 2.2 && < 2.3, directory >= 1.3.0.0 && < 2, @@ -291,9 +291,9 @@ criterion >= 1 && < 2, deepseq >= 1.4 && < 2, process >= 1.6 && < 2, - dataframe >= 3.3 && < 3.4, - dataframe-core >= 2.2 && < 2.3, - dataframe-operations >= 2.2 && < 2.3, + dataframe >= 3.4 && < 3.5, + dataframe-core >= 2.3 && < 2.4, + dataframe-operations >= 2.3 && < 2.4, random >= 1 && < 2, default-language: Haskell2010 ghc-options: @@ -377,16 +377,16 @@ build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.14, - dataframe >= 3.3 && < 3.4, - dataframe-core >= 2.2 && < 2.3, - dataframe-core >= 2.2 && < 2.3, + dataframe >= 3.4 && < 3.5, + dataframe-core >= 2.3 && < 2.4, + dataframe-core >= 2.3 && < 2.4, dataframe-csv >= 2.3 && < 2.4, dataframe-expr-serializer >= 1.2.0.1 && < 1.3, dataframe-fastcsv >= 1.4.0.1 && < 1.5, dataframe-json >= 1.2.0.1 && < 1.3, dataframe-lazy >= 2.3 && < 2.4, - dataframe-learn >= 2.2 && < 2.3, - dataframe-operations >= 2.2 && < 2.3, + dataframe-learn >= 2.3 && < 2.4, + dataframe-operations >= 2.3 && < 2.4, dataframe-parquet >= 1.5 && < 1.6, dataframe-parsing >= 2.2 && < 2.3, HUnit >= 1.6 && < 1.8, @@ -413,9 +413,9 @@ other-modules: Internal.PackedText build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, - dataframe >= 3.3 && < 3.4, - dataframe-core >= 2.2 && < 2.3, - dataframe-operations >= 2.2 && < 2.3, + dataframe >= 3.4 && < 3.5, + dataframe-core >= 2.3 && < 2.4, + dataframe-operations >= 2.3 && < 2.4, HUnit >= 1.6 && < 1.8, text >= 2.1 && < 3, vector >= 0.13 && < 0.15 diff --git a/tests/Learn/Ensembles.hs b/tests/Learn/Ensembles.hs --- a/tests/Learn/Ensembles.hs +++ b/tests/Learn/Ensembles.hs @@ -152,10 +152,58 @@ assertBool "best score high" (gsBestScore res > 0.99) assertEqual "all configs scored" 3 (length (gsAll res)) +{- | Logistic boosting must recover a known conditional probability, not merely +rank it. A leaf set to the mean gradient instead of @Σg/Σh@ understeps every +step by at least 4x, leaving probabilities shrunk toward the base rate while the +ranking — and so any accuracy or AUC check — still looks healthy. + +Each @x@ carries a fixed 20 rows of which exactly @round (20 * trueP x)@ are +positive, so the empirical conditional probability at every @x@ is 'trueP' and +the target is separable in rank but not in value. +-} +sigmoidCurveDF :: D.DataFrame +sigmoidCurveDF = + D.fromNamedColumns + [ ("x", DI.fromList (concatMap (replicate group . fst) cells)) + , ("label", DI.fromList (concatMap snd cells)) + ] + where + group = 20 :: Int + xs = [fromIntegral i / 10 - 1 | i <- [0 .. 20 :: Int]] :: [Double] + cells = [(x, labelsAt x) | x <- xs] + labelsAt x = + let k = round (fromIntegral group * trueP x) :: Int + in replicate k 1 ++ replicate (group - k) (0 :: Double) + +trueP :: Double -> Double +trueP x = 1 / (1 + exp (negate (4 * x))) + +testGBMCalibration :: Test +testGBMCalibration = TestCase $ do + let m = + fit + defaultGBConfig + { gbLoss = LogisticDeviance + , gbNEstimators = 100 + , gbLearningRate = 0.1 + , gbMaxDepth = 3 + } + (F.col @Double "label") + sigmoidCurveDF + probs = interpD sigmoidCurveDF (gbProbaExpr m) + truth = map trueP (interpD sigmoidCurveDF (F.col @Double "x")) + err = + sum (zipWith (\p t -> abs (p - t)) probs truth) + / fromIntegral (length probs) + assertBool + ("logistic boosting recovers the conditional probability " ++ show err) + (err < 0.03) + tests :: [Test] tests = [ testGBMRegression , testGBMStaged + , testGBMCalibration , testAdaBoost , testGMM , testDBSCAN diff --git a/tests/Learn/Metamorphic.hs b/tests/Learn/Metamorphic.hs --- a/tests/Learn/Metamorphic.hs +++ b/tests/Learn/Metamorphic.hs @@ -20,12 +20,12 @@ import qualified DataFrame.Functions as F import DataFrame.Internal.Column (TypedColumn (..), toVector) import qualified DataFrame.Internal.Column as DI -import DataFrame.Internal.Expression (Expr) +import DataFrame.Internal.Expression (Expr, getColumns) import DataFrame.Internal.Interpreter (interpret) +import DataFrame.DecisionTree.Regression (defaultRegTreeConfig) import DataFrame.LinearModel import DataFrame.Metrics -import DataFrame.Model (fit, predict) import DataFrame.Operations.Merge () -- Semigroup DataFrame (row concatenation) @@ -246,13 +246,6 @@ "column order: predictions on the same frame agree" (closeList 1e-7 p0 p1) --- Law: standardScaler output has mean ~0, std ~1 --------------------------- - -{- | The defining law of a standard scaler: after transforming, each scaled -column has sample mean ≈ 0 and (population) std ≈ 1. We recompute the moments -here in plain Haskell from the transformed frame — not via the scaler — so a -wrong denominator or a centring bug is caught. --} testStandardScalerLaw :: Test testStandardScalerLaw = TestCase $ do let cols = ["x1", "x2"] @@ -271,10 +264,6 @@ ) cols -{- | The scaler model's stored stats must match the data's own moments: a guard -against the scaler storing the wrong mean/std even if transform happens to -look plausible. Computed independently from the raw columns. --} testScalerStatsMatchData :: Test testScalerStatsMatchData = TestCase $ do let scaler = standardScaler ["x1", "x2"] baseDF @@ -291,11 +280,6 @@ "scaler stds match data" (closeList 1e-9 (VU.toList (smStds scaler)) [sd1, sd2]) --- Metric laws -------------------------------------------------------------- - -{- | Perfect prediction → accuracy exactly 1.0; a single deliberate miss drops it -below 1. Pins both ends so a metric that ignores its inputs can't pass. --} testAccuracyLaw :: Test testAccuracyLaw = TestCase $ do let truth = VU.fromList [0, 1, 2, 1, 0, 2] @@ -309,9 +293,6 @@ "accuracy in [0,1]" (let a = accuracy oneWrong truth in a >= 0 && a <= 1) -{- | Accuracy is permutation-invariant: applying the same permutation to preds -and truth leaves it unchanged. Both vectors are genuinely reordered. --} testAccuracyPermInvariant :: Test testAccuracyPermInvariant = TestCase $ do let preds = VU.fromList [0, 0, 1, 1, 2, 2, 1, 0] @@ -322,10 +303,6 @@ -- Sanity: the metric is non-trivial here (not 0 or 1), so invariance is meaningful. assertBool "accuracy: non-degenerate baseline" (a0 > 0 && a0 < 1) -{- | r² of a perfect fit is exactly 1; r² of predicting the constant mean is -exactly 0. These are the two anchor points of the R² definition. Catches a -swapped SS_res/SS_tot or a wrong sign. --} testR2Anchors :: Test testR2Anchors = TestCase $ do let truth = VU.fromList [1, 3, 2, 8, 5, 4] @@ -348,9 +325,37 @@ assertBool "r2 of exact linear fit ~ 1" (close 1e-9 score 1.0) assertBool "rmse of exact linear fit ~ 0" (err < 1e-6) +testTargetNeverAFeature :: Test +testTargetNeverAFeature = TestCase $ do + let n = 40 :: Int + leakDF = + D.fromNamedColumns + [ ("y", DI.fromList [if even i then 1.0 else 0.0 :: Double | i <- [0 .. n - 1]]) + , ("a", DI.fromList [fromIntegral (i `div` 4) :: Double | i <- [0 .. n - 1]]) + , ("b", DI.fromList [fromIntegral (i `mod` 3) :: Double | i <- [0 .. n - 1]]) + ] + assertNoTarget name expr = + assertBool + ( name + ++ ": target 'y' must not appear in the fitted Expr (got " + ++ show expr + ++ ")" + ) + ("y" `notElem` getColumns expr) + assertNoTarget + "linear" + (predict (fit defaultLinearConfig (F.col @Double "y") leakDF)) + assertNoTarget + "regression tree" + (predict (fit defaultRegTreeConfig (F.col @Double "y") leakDF)) + assertNoTarget + "classification tree" + (predict (fit D.defaultTreeConfig (F.col @Double "y") leakDF)) + tests :: [Test] tests = - [ testDuplicateRows + [ testTargetNeverAFeature + , testDuplicateRows , testPermuteRows , testScaleFeature , testRenameColumns diff --git a/tests/Learn/Synthesis.hs b/tests/Learn/Synthesis.hs --- a/tests/Learn/Synthesis.hs +++ b/tests/Learn/Synthesis.hs @@ -7,8 +7,9 @@ -} module Learn.Synthesis (tests) where +import Assertions (assertExpectException) +import qualified Data.Text as T import qualified DataFrame as D -import DataFrame.Model (fit) import DataFrame.Synthesis import Test.HUnit @@ -73,9 +74,48 @@ (D.prettyPrint (sfExpr a)) (D.prettyPrint (sfExpr b)) +{- | A wide frame at the default 'synMaxSize' refuses instead of exhausting the +heap. 'synBankCap' caps what is kept, not what is generated, so the layers past +size 4 used to allocate tens of gigabytes and kill the process — which no test +can catch, because there is no process left to fail. +-} +refusesOversizedSearch :: Test +refusesOversizedSearch = + TestCase + ( assertExpectException + "[Error Case]" + "synMaxAllocBytes" + ( print + (D.prettyPrint (sfExpr (fit defaultSynthesisConfig (D.col @Double "y") wide))) + ) + ) + +-- | The same frame is fine once the search is small enough to fit the budget. +acceptsSmallSearch :: Test +acceptsSmallSearch = TestCase $ do + let cfg = defaultSynthesisConfig{synMaxSize = 3} + m = fit cfg (D.col @Double "y") wide + assertBool "a size-3 search over the wide frame returns" (sfScore m >= -1.0) + +-- | 12 features over 3000 rows: the shape that killed the kernel. +wide :: D.DataFrame +wide = + D.fromNamedColumns + ( ("y", D.fromList (map (\i -> fromIntegral (i `mod` 7) :: Double) idx)) + : [ ( "f" <> T.pack (show c) + , D.fromList (map (\i -> fromIntegral ((i * c) `mod` 13) :: Double) idx) + ) + | c <- [1 .. 12 :: Int] + ] + ) + where + idx = [0 .. 2999 :: Int] + tests :: [Test] tests = - [ recoversQuadratic + [ refusesOversizedSearch + , acceptsSmallSearch + , recoversQuadratic , exactRecoveryMSE , recoversRatio , distinctFeatures diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -107,6 +107,7 @@ ++ Operations.Shuffle.tests ++ Operations.Sort.tests ++ Operations.Statistics.tests + ++ Monad.hunitTests ++ Operations.Subset.hunitTests ++ Operations.Take.tests ++ Operations.Typing.tests diff --git a/tests/Monad.hs b/tests/Monad.hs --- a/tests/Monad.hs +++ b/tests/Monad.hs @@ -1,10 +1,17 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + module Monad where +import qualified Data.Text as T import qualified DataFrame as D +import qualified DataFrame.Functions as F +import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame import DataFrame.Monad import GenDataFrame () import System.Random +import qualified Test.HUnit as H import Test.QuickCheck import Test.QuickCheck.Monadic @@ -29,3 +36,50 @@ tests :: [DataFrame -> Gen (Gen Property)] tests = [prop_sampleM] + +-- Column-shaped verbs: 'dropM' drops rows, so these had no monadic spelling. + +verbFixture :: DataFrame +verbFixture = + D.fromNamedColumns + [ ("A", DI.fromList ([3, 1, 2] :: [Int])) + , ("B", DI.fromList (["x", "y", "z"] :: [T.Text])) + , ("C", DI.fromList ([1.0, 2.0, 3.0] :: [Double])) + ] + +selectMKeepsColumns :: H.Test +selectMKeepsColumns = + H.TestCase + ( H.assertEqual + "selectM keeps only the named columns" + ["A", "B"] + (D.columnNames (execFrameM verbFixture (selectM ["A", "B"]))) + ) + +excludeMDropsColumns :: H.Test +excludeMDropsColumns = + H.TestCase + ( H.assertEqual + "excludeM drops the named columns" + ["A", "C"] + (D.columnNames (execFrameM verbFixture (excludeM ["B"]))) + ) + +sortByMOrdersRows :: H.Test +sortByMOrdersRows = + H.TestCase + ( H.assertEqual + "sortByM sorts ascending on A" + [1, 2, 3] + ( D.columnAsList @Int + (F.col @Int "A") + (execFrameM verbFixture (sortByM [Asc (F.col @Int "A")])) + ) + ) + +hunitTests :: [H.Test] +hunitTests = + [ H.TestLabel "selectMKeepsColumns" selectMKeepsColumns + , H.TestLabel "excludeMDropsColumns" excludeMDropsColumns + , H.TestLabel "sortByMOrdersRows" sortByMOrdersRows + ] diff --git a/tests/Operations/Apply.hs b/tests/Operations/Apply.hs --- a/tests/Operations/Apply.hs +++ b/tests/Operations/Apply.hs @@ -255,28 +255,29 @@ imputeOnNonOptional :: Test imputeOnNonOptional = TestCase - ( assertEqual - "impute is a no-op on a non-nullable column" - imputeData - (impute (F.col @(Maybe Int) "plain") 0 imputeData) + ( assertExpectException + "[Error Case]" + "impute" + (print $ impute (F.col @(Maybe Int) "plain") 0 imputeData) ) -imputePlainNoOp :: Test -imputePlainNoOp = +-- | Only a column reference can be imputed; a compound expression throws. +imputeCompoundExprThrows :: Test +imputeCompoundExprThrows = TestCase - ( assertEqual - "impute with non-Maybe expr is always a no-op" - imputeData - (impute (F.col @Int "plain") 0 imputeData) + ( assertExpectException + "[Error Case]" + "column reference" + (print $ impute (F.lit (Just (1 :: Int))) 0 imputeData) ) -imputeWithPlainNoOp :: Test -imputeWithPlainNoOp = +imputeWithCompoundExprThrows :: Test +imputeWithCompoundExprThrows = TestCase - ( assertEqual - "imputeWith with non-Maybe expr is always a no-op" - imputeData - (imputeWith id (F.col @Int "plain") imputeData) + ( assertExpectException + "[Error Case]" + "column reference" + (print $ imputeWith id (F.lit (Just (1 :: Int))) imputeData) ) tests :: [Test] @@ -299,6 +300,6 @@ , TestLabel "imputeHappyPath" imputeHappyPath , TestLabel "imputeColumnNotFound" imputeColumnNotFound , TestLabel "imputeOnNonOptional" imputeOnNonOptional - , TestLabel "imputePlainNoOp" imputePlainNoOp - , TestLabel "imputeWithPlainNoOp" imputeWithPlainNoOp + , TestLabel "imputeCompoundExprThrows" imputeCompoundExprThrows + , TestLabel "imputeWithCompoundExprThrows" imputeWithCompoundExprThrows ] diff --git a/tests/Operations/Core.hs b/tests/Operations/Core.hs --- a/tests/Operations/Core.hs +++ b/tests/Operations/Core.hs @@ -2,8 +2,12 @@ module Operations.Core where +import qualified Data.Text as T + +import Assertions (assertExpectException) import qualified DataFrame as D import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.Row (Any (..)) import Test.HUnit @@ -29,5 +33,84 @@ ) ) +fromRowsThrowsOnTypeMismatch :: Test +fromRowsThrowsOnTypeMismatch = + TestCase + ( assertExpectException + "[Error Case]" + "fromRows" + ( print $ + D.fromRows + ["A"] + [ [D.toAny (1 :: Int)] + , [D.toAny ('x' :: Char)] + , [D.toAny (3 :: Int)] + ] + ) + ) + +fromRowsThrowsOnShortRow :: Test +fromRowsThrowsOnShortRow = + TestCase + ( assertExpectException + "[Error Case]" + "fromRows" + ( print $ + D.fromRows + ["A", "B"] + [ [D.toAny (1 :: Int), D.toAny (10 :: Int)] + , [D.toAny (2 :: Int)] + ] + ) + ) + +-- | A null keeps its row: the column stays full length and values stay put. +fromRowsKeepsNullsInPlace :: Test +fromRowsKeepsNullsInPlace = + TestCase + ( assertEqual + "null cell preserves row alignment" + ( D.fromNamedColumns + [("A", DI.fromList ([Just 1, Nothing, Just 3] :: [Maybe Int]))] + ) + (D.fromRows ["A"] [[D.toAny (1 :: Int)], [Null], [D.toAny (3 :: Int)]]) + ) + +{- | An all-null column has as many rows as it was given. Collapsing it to an +empty column silently truncates the frame. +-} +fromRowsAllNullColumnKeepsRows :: Test +fromRowsAllNullColumnKeepsRows = + TestCase + ( assertEqual + "all-null column keeps its rows" + 3 + (D.nRows (D.fromRows ["A"] [[Null], [Null], [Null]])) + ) + +{- | A frame with a null survives the round trip at full length. Guards the +alignment invariant through 'toRowList' as well as 'fromRows'. +-} +fromRowsRoundTripsWithNulls :: Test +fromRowsRoundTripsWithNulls = + TestCase + ( let df = + D.fromNamedColumns + [ ("A", DI.fromList ([Just 1, Nothing, Just 3] :: [Maybe Int])) + , ("B", DI.fromList (["x", "y", "z"] :: [T.Text])) + ] + in assertEqual + "round trip through rows preserves the frame" + df + (D.fromRows (D.columnNames df) (map (map snd) (D.toRowList df))) + ) + tests :: [Test] -tests = [TestLabel "createsDataFrameFromRows" createsDataFrameFromRows] +tests = + [ TestLabel "createsDataFrameFromRows" createsDataFrameFromRows + , TestLabel "fromRowsThrowsOnTypeMismatch" fromRowsThrowsOnTypeMismatch + , TestLabel "fromRowsThrowsOnShortRow" fromRowsThrowsOnShortRow + , TestLabel "fromRowsKeepsNullsInPlace" fromRowsKeepsNullsInPlace + , TestLabel "fromRowsAllNullColumnKeepsRows" fromRowsAllNullColumnKeepsRows + , TestLabel "fromRowsRoundTripsWithNulls" fromRowsRoundTripsWithNulls + ]