dataframe 3.3.0.0 → 3.4.0.0
raw patch · 9 files changed
+312/−72 lines, 9 filesdep ~dataframedep ~dataframe-coredep ~dataframe-learnPVP ok
version bump matches the API change (PVP)
Dependency ranges changed: dataframe, dataframe-core, dataframe-learn, dataframe-operations
API changes (from Hackage documentation)
Files
- CHANGELOG.md +8/−0
- dataframe.cabal +25/−25
- tests/Learn/Ensembles.hs +48/−0
- tests/Learn/Metamorphic.hs +31/−26
- tests/Learn/Synthesis.hs +42/−2
- tests/Main.hs +1/−0
- tests/Monad.hs +54/−0
- tests/Operations/Apply.hs +19/−18
- tests/Operations/Core.hs +84/−1
CHANGELOG.md view
@@ -1,5 +1,13 @@ # Revision history for dataframe +## 3.4.0.0++* `impute` on a non-nullable expression is now fails and throws when given a column with the wrong type.+* `fromRows` throws on a cell whose type differs from its column's, and on a row too short to reach a column.+* `F.add`, `F.sub`, `F.mult`, `F.divide`, `F.prettyPrint` are now visible.+* `F.isNull` / `F.isNotNull` as aliases for `isNothing` / `isJust`.+* `DataFrame.Monad` now implements `selectM`, `excludeM` and `sortByM`.+ ## 3.3.0.0 ### Breaking changes
dataframe.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe-version: 3.3.0.0+version: 3.4.0.0 synopsis: A fast, safe, and intuitive DataFrame library. description: A fast, safe, and intuitive DataFrame library for exploratory data analysis.@@ -128,13 +128,13 @@ DataFrame.Typed.Record, DataFrame.Typed.Generic build-depends: base >= 4 && <5,- dataframe-core >= 2.2 && < 2.3,+ dataframe-core >= 2.3 && < 2.4, dataframe-json >= 1.2.0.1 && < 1.3, dataframe-expr-serializer >= 1.2.0.1 && < 1.3,- dataframe-operations >= 2.2 && < 2.3,+ dataframe-operations >= 2.3 && < 2.4, dataframe-parsing >= 2.2 && < 2.3, dataframe-viz >= 1.3 && < 1.4,- dataframe-learn >= 2.2 && < 2.3+ dataframe-learn >= 2.3 && < 2.4 if !flag(no-csv) reexported-modules: DataFrame.IO.CSV,@@ -203,12 +203,12 @@ buildable: False build-depends: base >= 4 && < 5,- dataframe-core >= 2.2 && < 2.3,+ dataframe-core >= 2.3 && < 2.4, dataframe-expr-serializer >= 1.2.0.1 && < 1.3, dataframe-csv >= 2.3 && < 2.4, dataframe-json >= 1.2.0.1 && < 1.3, dataframe-lazy >= 2.3 && < 2.4,- dataframe-operations >= 2.2 && < 2.3,+ dataframe-operations >= 2.3 && < 2.4, dataframe-parquet >= 1.5 && < 1.6, dataframe-parsing >= 2.2 && < 2.3, text >= 2.1 && < 3,@@ -223,8 +223,8 @@ import: warnings main-is: Benchmark.hs build-depends: base >= 4 && < 5,- dataframe >= 3.3 && < 3.4,- dataframe-operations >= 2.2 && < 2.3,+ dataframe >= 3.4 && < 3.5,+ dataframe-operations >= 2.3 && < 2.4, random >= 1 && < 2, time >= 1.12 && < 2, vector >= 0.13 && < 0.15,@@ -236,10 +236,10 @@ import: warnings main-is: Synthesis.hs build-depends: base >= 4 && < 5,- dataframe >= 3.3 && < 3.4,- dataframe-core >= 2.2 && < 2.3,- dataframe-learn >= 2.2 && < 2.3,- dataframe-operations >= 2.2 && < 2.3,+ dataframe >= 3.4 && < 3.5,+ dataframe-core >= 2.3 && < 2.4,+ dataframe-learn >= 2.3 && < 2.4,+ dataframe-operations >= 2.3 && < 2.4, random >= 1 && < 2, text >= 2.1 && < 3 hs-source-dirs: app@@ -270,8 +270,8 @@ build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, containers >= 0.6.7 && < 0.10,- dataframe >= 3.3 && < 3.4,- dataframe-core >= 2.2 && < 2.3,+ dataframe >= 3.4 && < 3.5,+ dataframe-core >= 2.3 && < 2.4, dataframe-lazy >= 2.3 && < 2.4, dataframe-parsing >= 2.2 && < 2.3, directory >= 1.3.0.0 && < 2,@@ -291,9 +291,9 @@ criterion >= 1 && < 2, deepseq >= 1.4 && < 2, process >= 1.6 && < 2,- dataframe >= 3.3 && < 3.4,- dataframe-core >= 2.2 && < 2.3,- dataframe-operations >= 2.2 && < 2.3,+ dataframe >= 3.4 && < 3.5,+ dataframe-core >= 2.3 && < 2.4,+ dataframe-operations >= 2.3 && < 2.4, random >= 1 && < 2, default-language: Haskell2010 ghc-options:@@ -377,16 +377,16 @@ build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.14,- dataframe >= 3.3 && < 3.4,- dataframe-core >= 2.2 && < 2.3,- dataframe-core >= 2.2 && < 2.3,+ dataframe >= 3.4 && < 3.5,+ dataframe-core >= 2.3 && < 2.4,+ dataframe-core >= 2.3 && < 2.4, dataframe-csv >= 2.3 && < 2.4, dataframe-expr-serializer >= 1.2.0.1 && < 1.3, dataframe-fastcsv >= 1.4.0.1 && < 1.5, dataframe-json >= 1.2.0.1 && < 1.3, dataframe-lazy >= 2.3 && < 2.4,- dataframe-learn >= 2.2 && < 2.3,- dataframe-operations >= 2.2 && < 2.3,+ dataframe-learn >= 2.3 && < 2.4,+ dataframe-operations >= 2.3 && < 2.4, dataframe-parquet >= 1.5 && < 1.6, dataframe-parsing >= 2.2 && < 2.3, HUnit >= 1.6 && < 1.8,@@ -413,9 +413,9 @@ other-modules: Internal.PackedText build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14,- dataframe >= 3.3 && < 3.4,- dataframe-core >= 2.2 && < 2.3,- dataframe-operations >= 2.2 && < 2.3,+ dataframe >= 3.4 && < 3.5,+ dataframe-core >= 2.3 && < 2.4,+ dataframe-operations >= 2.3 && < 2.4, HUnit >= 1.6 && < 1.8, text >= 2.1 && < 3, vector >= 0.13 && < 0.15
tests/Learn/Ensembles.hs view
@@ -152,10 +152,58 @@ assertBool "best score high" (gsBestScore res > 0.99) assertEqual "all configs scored" 3 (length (gsAll res)) +{- | Logistic boosting must recover a known conditional probability, not merely+rank it. A leaf set to the mean gradient instead of @Σg/Σh@ understeps every+step by at least 4x, leaving probabilities shrunk toward the base rate while the+ranking — and so any accuracy or AUC check — still looks healthy.++Each @x@ carries a fixed 20 rows of which exactly @round (20 * trueP x)@ are+positive, so the empirical conditional probability at every @x@ is 'trueP' and+the target is separable in rank but not in value.+-}+sigmoidCurveDF :: D.DataFrame+sigmoidCurveDF =+ D.fromNamedColumns+ [ ("x", DI.fromList (concatMap (replicate group . fst) cells))+ , ("label", DI.fromList (concatMap snd cells))+ ]+ where+ group = 20 :: Int+ xs = [fromIntegral i / 10 - 1 | i <- [0 .. 20 :: Int]] :: [Double]+ cells = [(x, labelsAt x) | x <- xs]+ labelsAt x =+ let k = round (fromIntegral group * trueP x) :: Int+ in replicate k 1 ++ replicate (group - k) (0 :: Double)++trueP :: Double -> Double+trueP x = 1 / (1 + exp (negate (4 * x)))++testGBMCalibration :: Test+testGBMCalibration = TestCase $ do+ let m =+ fit+ defaultGBConfig+ { gbLoss = LogisticDeviance+ , gbNEstimators = 100+ , gbLearningRate = 0.1+ , gbMaxDepth = 3+ }+ (F.col @Double "label")+ sigmoidCurveDF+ probs = interpD sigmoidCurveDF (gbProbaExpr m)+ truth = map trueP (interpD sigmoidCurveDF (F.col @Double "x"))+ err =+ sum (zipWith (\p t -> abs (p - t)) probs truth)+ / fromIntegral (length probs)+ assertBool+ ("logistic boosting recovers the conditional probability " ++ show err)+ (err < 0.03)+ tests :: [Test] tests = [ testGBMRegression , testGBMStaged+ , testGBMCalibration , testAdaBoost , testGMM , testDBSCAN
tests/Learn/Metamorphic.hs view
@@ -20,12 +20,12 @@ import qualified DataFrame.Functions as F import DataFrame.Internal.Column (TypedColumn (..), toVector) import qualified DataFrame.Internal.Column as DI-import DataFrame.Internal.Expression (Expr)+import DataFrame.Internal.Expression (Expr, getColumns) import DataFrame.Internal.Interpreter (interpret) +import DataFrame.DecisionTree.Regression (defaultRegTreeConfig) import DataFrame.LinearModel import DataFrame.Metrics-import DataFrame.Model (fit, predict) import DataFrame.Operations.Merge () -- Semigroup DataFrame (row concatenation)@@ -246,13 +246,6 @@ "column order: predictions on the same frame agree" (closeList 1e-7 p0 p1) --- Law: standardScaler output has mean ~0, std ~1 -----------------------------{- | The defining law of a standard scaler: after transforming, each scaled-column has sample mean ≈ 0 and (population) std ≈ 1. We recompute the moments-here in plain Haskell from the transformed frame — not via the scaler — so a-wrong denominator or a centring bug is caught.--} testStandardScalerLaw :: Test testStandardScalerLaw = TestCase $ do let cols = ["x1", "x2"]@@ -271,10 +264,6 @@ ) cols -{- | The scaler model's stored stats must match the data's own moments: a guard-against the scaler storing the wrong mean/std even if transform happens to-look plausible. Computed independently from the raw columns.--} testScalerStatsMatchData :: Test testScalerStatsMatchData = TestCase $ do let scaler = standardScaler ["x1", "x2"] baseDF@@ -291,11 +280,6 @@ "scaler stds match data" (closeList 1e-9 (VU.toList (smStds scaler)) [sd1, sd2]) --- Metric laws ----------------------------------------------------------------{- | Perfect prediction → accuracy exactly 1.0; a single deliberate miss drops it-below 1. Pins both ends so a metric that ignores its inputs can't pass.--} testAccuracyLaw :: Test testAccuracyLaw = TestCase $ do let truth = VU.fromList [0, 1, 2, 1, 0, 2]@@ -309,9 +293,6 @@ "accuracy in [0,1]" (let a = accuracy oneWrong truth in a >= 0 && a <= 1) -{- | Accuracy is permutation-invariant: applying the same permutation to preds-and truth leaves it unchanged. Both vectors are genuinely reordered.--} testAccuracyPermInvariant :: Test testAccuracyPermInvariant = TestCase $ do let preds = VU.fromList [0, 0, 1, 1, 2, 2, 1, 0]@@ -322,10 +303,6 @@ -- Sanity: the metric is non-trivial here (not 0 or 1), so invariance is meaningful. assertBool "accuracy: non-degenerate baseline" (a0 > 0 && a0 < 1) -{- | r² of a perfect fit is exactly 1; r² of predicting the constant mean is-exactly 0. These are the two anchor points of the R² definition. Catches a-swapped SS_res/SS_tot or a wrong sign.--} testR2Anchors :: Test testR2Anchors = TestCase $ do let truth = VU.fromList [1, 3, 2, 8, 5, 4]@@ -348,9 +325,37 @@ assertBool "r2 of exact linear fit ~ 1" (close 1e-9 score 1.0) assertBool "rmse of exact linear fit ~ 0" (err < 1e-6) +testTargetNeverAFeature :: Test+testTargetNeverAFeature = TestCase $ do+ let n = 40 :: Int+ leakDF =+ D.fromNamedColumns+ [ ("y", DI.fromList [if even i then 1.0 else 0.0 :: Double | i <- [0 .. n - 1]])+ , ("a", DI.fromList [fromIntegral (i `div` 4) :: Double | i <- [0 .. n - 1]])+ , ("b", DI.fromList [fromIntegral (i `mod` 3) :: Double | i <- [0 .. n - 1]])+ ]+ assertNoTarget name expr =+ assertBool+ ( name+ ++ ": target 'y' must not appear in the fitted Expr (got "+ ++ show expr+ ++ ")"+ )+ ("y" `notElem` getColumns expr)+ assertNoTarget+ "linear"+ (predict (fit defaultLinearConfig (F.col @Double "y") leakDF))+ assertNoTarget+ "regression tree"+ (predict (fit defaultRegTreeConfig (F.col @Double "y") leakDF))+ assertNoTarget+ "classification tree"+ (predict (fit D.defaultTreeConfig (F.col @Double "y") leakDF))+ tests :: [Test] tests =- [ testDuplicateRows+ [ testTargetNeverAFeature+ , testDuplicateRows , testPermuteRows , testScaleFeature , testRenameColumns
tests/Learn/Synthesis.hs view
@@ -7,8 +7,9 @@ -} module Learn.Synthesis (tests) where +import Assertions (assertExpectException)+import qualified Data.Text as T import qualified DataFrame as D-import DataFrame.Model (fit) import DataFrame.Synthesis import Test.HUnit@@ -73,9 +74,48 @@ (D.prettyPrint (sfExpr a)) (D.prettyPrint (sfExpr b)) +{- | A wide frame at the default 'synMaxSize' refuses instead of exhausting the+heap. 'synBankCap' caps what is kept, not what is generated, so the layers past+size 4 used to allocate tens of gigabytes and kill the process — which no test+can catch, because there is no process left to fail.+-}+refusesOversizedSearch :: Test+refusesOversizedSearch =+ TestCase+ ( assertExpectException+ "[Error Case]"+ "synMaxAllocBytes"+ ( print+ (D.prettyPrint (sfExpr (fit defaultSynthesisConfig (D.col @Double "y") wide)))+ )+ )++-- | The same frame is fine once the search is small enough to fit the budget.+acceptsSmallSearch :: Test+acceptsSmallSearch = TestCase $ do+ let cfg = defaultSynthesisConfig{synMaxSize = 3}+ m = fit cfg (D.col @Double "y") wide+ assertBool "a size-3 search over the wide frame returns" (sfScore m >= -1.0)++-- | 12 features over 3000 rows: the shape that killed the kernel.+wide :: D.DataFrame+wide =+ D.fromNamedColumns+ ( ("y", D.fromList (map (\i -> fromIntegral (i `mod` 7) :: Double) idx))+ : [ ( "f" <> T.pack (show c)+ , D.fromList (map (\i -> fromIntegral ((i * c) `mod` 13) :: Double) idx)+ )+ | c <- [1 .. 12 :: Int]+ ]+ )+ where+ idx = [0 .. 2999 :: Int]+ tests :: [Test] tests =- [ recoversQuadratic+ [ refusesOversizedSearch+ , acceptsSmallSearch+ , recoversQuadratic , exactRecoveryMSE , recoversRatio , distinctFeatures
tests/Main.hs view
@@ -107,6 +107,7 @@ ++ Operations.Shuffle.tests ++ Operations.Sort.tests ++ Operations.Statistics.tests+ ++ Monad.hunitTests ++ Operations.Subset.hunitTests ++ Operations.Take.tests ++ Operations.Typing.tests
tests/Monad.hs view
@@ -1,10 +1,17 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE TypeApplications #-}+ module Monad where +import qualified Data.Text as T import qualified DataFrame as D+import qualified DataFrame.Functions as F+import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame import DataFrame.Monad import GenDataFrame () import System.Random+import qualified Test.HUnit as H import Test.QuickCheck import Test.QuickCheck.Monadic @@ -29,3 +36,50 @@ tests :: [DataFrame -> Gen (Gen Property)] tests = [prop_sampleM]++-- Column-shaped verbs: 'dropM' drops rows, so these had no monadic spelling.++verbFixture :: DataFrame+verbFixture =+ D.fromNamedColumns+ [ ("A", DI.fromList ([3, 1, 2] :: [Int]))+ , ("B", DI.fromList (["x", "y", "z"] :: [T.Text]))+ , ("C", DI.fromList ([1.0, 2.0, 3.0] :: [Double]))+ ]++selectMKeepsColumns :: H.Test+selectMKeepsColumns =+ H.TestCase+ ( H.assertEqual+ "selectM keeps only the named columns"+ ["A", "B"]+ (D.columnNames (execFrameM verbFixture (selectM ["A", "B"])))+ )++excludeMDropsColumns :: H.Test+excludeMDropsColumns =+ H.TestCase+ ( H.assertEqual+ "excludeM drops the named columns"+ ["A", "C"]+ (D.columnNames (execFrameM verbFixture (excludeM ["B"])))+ )++sortByMOrdersRows :: H.Test+sortByMOrdersRows =+ H.TestCase+ ( H.assertEqual+ "sortByM sorts ascending on A"+ [1, 2, 3]+ ( D.columnAsList @Int+ (F.col @Int "A")+ (execFrameM verbFixture (sortByM [Asc (F.col @Int "A")]))+ )+ )++hunitTests :: [H.Test]+hunitTests =+ [ H.TestLabel "selectMKeepsColumns" selectMKeepsColumns+ , H.TestLabel "excludeMDropsColumns" excludeMDropsColumns+ , H.TestLabel "sortByMOrdersRows" sortByMOrdersRows+ ]
tests/Operations/Apply.hs view
@@ -255,28 +255,29 @@ imputeOnNonOptional :: Test imputeOnNonOptional = TestCase- ( assertEqual- "impute is a no-op on a non-nullable column"- imputeData- (impute (F.col @(Maybe Int) "plain") 0 imputeData)+ ( assertExpectException+ "[Error Case]"+ "impute"+ (print $ impute (F.col @(Maybe Int) "plain") 0 imputeData) ) -imputePlainNoOp :: Test-imputePlainNoOp =+-- | Only a column reference can be imputed; a compound expression throws.+imputeCompoundExprThrows :: Test+imputeCompoundExprThrows = TestCase- ( assertEqual- "impute with non-Maybe expr is always a no-op"- imputeData- (impute (F.col @Int "plain") 0 imputeData)+ ( assertExpectException+ "[Error Case]"+ "column reference"+ (print $ impute (F.lit (Just (1 :: Int))) 0 imputeData) ) -imputeWithPlainNoOp :: Test-imputeWithPlainNoOp =+imputeWithCompoundExprThrows :: Test+imputeWithCompoundExprThrows = TestCase- ( assertEqual- "imputeWith with non-Maybe expr is always a no-op"- imputeData- (imputeWith id (F.col @Int "plain") imputeData)+ ( assertExpectException+ "[Error Case]"+ "column reference"+ (print $ imputeWith id (F.lit (Just (1 :: Int))) imputeData) ) tests :: [Test]@@ -299,6 +300,6 @@ , TestLabel "imputeHappyPath" imputeHappyPath , TestLabel "imputeColumnNotFound" imputeColumnNotFound , TestLabel "imputeOnNonOptional" imputeOnNonOptional- , TestLabel "imputePlainNoOp" imputePlainNoOp- , TestLabel "imputeWithPlainNoOp" imputeWithPlainNoOp+ , TestLabel "imputeCompoundExprThrows" imputeCompoundExprThrows+ , TestLabel "imputeWithCompoundExprThrows" imputeWithCompoundExprThrows ]
tests/Operations/Core.hs view
@@ -2,8 +2,12 @@ module Operations.Core where +import qualified Data.Text as T++import Assertions (assertExpectException) import qualified DataFrame as D import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.Row (Any (..)) import Test.HUnit @@ -29,5 +33,84 @@ ) ) +fromRowsThrowsOnTypeMismatch :: Test+fromRowsThrowsOnTypeMismatch =+ TestCase+ ( assertExpectException+ "[Error Case]"+ "fromRows"+ ( print $+ D.fromRows+ ["A"]+ [ [D.toAny (1 :: Int)]+ , [D.toAny ('x' :: Char)]+ , [D.toAny (3 :: Int)]+ ]+ )+ )++fromRowsThrowsOnShortRow :: Test+fromRowsThrowsOnShortRow =+ TestCase+ ( assertExpectException+ "[Error Case]"+ "fromRows"+ ( print $+ D.fromRows+ ["A", "B"]+ [ [D.toAny (1 :: Int), D.toAny (10 :: Int)]+ , [D.toAny (2 :: Int)]+ ]+ )+ )++-- | A null keeps its row: the column stays full length and values stay put.+fromRowsKeepsNullsInPlace :: Test+fromRowsKeepsNullsInPlace =+ TestCase+ ( assertEqual+ "null cell preserves row alignment"+ ( D.fromNamedColumns+ [("A", DI.fromList ([Just 1, Nothing, Just 3] :: [Maybe Int]))]+ )+ (D.fromRows ["A"] [[D.toAny (1 :: Int)], [Null], [D.toAny (3 :: Int)]])+ )++{- | An all-null column has as many rows as it was given. Collapsing it to an+empty column silently truncates the frame.+-}+fromRowsAllNullColumnKeepsRows :: Test+fromRowsAllNullColumnKeepsRows =+ TestCase+ ( assertEqual+ "all-null column keeps its rows"+ 3+ (D.nRows (D.fromRows ["A"] [[Null], [Null], [Null]]))+ )++{- | A frame with a null survives the round trip at full length. Guards the+alignment invariant through 'toRowList' as well as 'fromRows'.+-}+fromRowsRoundTripsWithNulls :: Test+fromRowsRoundTripsWithNulls =+ TestCase+ ( let df =+ D.fromNamedColumns+ [ ("A", DI.fromList ([Just 1, Nothing, Just 3] :: [Maybe Int]))+ , ("B", DI.fromList (["x", "y", "z"] :: [T.Text]))+ ]+ in assertEqual+ "round trip through rows preserves the frame"+ df+ (D.fromRows (D.columnNames df) (map (map snd) (D.toRowList df)))+ )+ tests :: [Test]-tests = [TestLabel "createsDataFrameFromRows" createsDataFrameFromRows]+tests =+ [ TestLabel "createsDataFrameFromRows" createsDataFrameFromRows+ , TestLabel "fromRowsThrowsOnTypeMismatch" fromRowsThrowsOnTypeMismatch+ , TestLabel "fromRowsThrowsOnShortRow" fromRowsThrowsOnShortRow+ , TestLabel "fromRowsKeepsNullsInPlace" fromRowsKeepsNullsInPlace+ , TestLabel "fromRowsAllNullColumnKeepsRows" fromRowsAllNullColumnKeepsRows+ , TestLabel "fromRowsRoundTripsWithNulls" fromRowsRoundTripsWithNulls+ ]