dataframe-learn 2.3.0.0 → 2.4.0.0
raw patch · 4 files changed
+141/−124 lines, 4 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- dataframe-learn.cabal +1/−1
- src-internal/DataFrame/DecisionTree/Numeric.hs +32/−11
- tests-internal/DecisionTree.hs +108/−12
- tests-internal/LinearSolver.hs +0/−100
dataframe-learn.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe-learn-version: 2.3.0.0+version: 2.4.0.0 synopsis: Interpretable, expression-returning machine learning for the dataframe ecosystem. description: A small scikit-learn-style ML library where every model returns both an
src-internal/DataFrame/DecisionTree/Numeric.hs view
@@ -6,8 +6,10 @@ {-# LANGUAGE TypeApplications #-} {- | Numeric split candidates: per-column Double expressions, arithmetic-expansion, and threshold conditions. 'numericCondVecs' materializes the-pool with one interpret per distinct expression.+expansion, and threshold conditions. Nullable columns also contribute an+@isNothing@ candidate, so missingness is splittable directly rather than+only via the null-routing of value splits. 'numericCondVecs' materializes+the pool with one interpret per distinct expression. -} module DataFrame.DecisionTree.Numeric ( NumExpr (..),@@ -16,13 +18,14 @@ combineNumExprs, numericConditions, generateNumericConds,+ missingnessConditions, percentilesOf, numericCondVecs, numericExprsWithTerms, numericCols, ) where -import DataFrame.DecisionTree.CondVec (CondVec (..))+import DataFrame.DecisionTree.CondVec (CondVec (..), materializeCondVec) import DataFrame.DecisionTree.Types (SynthConfig (..), TreeConfig (..)) import qualified DataFrame.Functions as F import DataFrame.Internal.Column@@ -33,7 +36,7 @@ import DataFrame.Operators import Data.List (sort)-import Data.Maybe (fromMaybe)+import Data.Maybe (catMaybes, mapMaybe) import qualified Data.Set as Set import qualified Data.Text as T import Data.Type.Equality (testEquality, (:~:) (..))@@ -95,14 +98,30 @@ numericConditions = generateNumericConds generateNumericConds :: TreeConfig -> DataFrame -> [Expr Bool]-generateNumericConds cfg df = do- expr <- numericExprsWithTerms (synthConfig cfg) df- threshold <- numericThresholds cfg df expr- condsFromExpr expr threshold+generateNumericConds cfg df = thresholdConds ++ missingnessConditions df+ where+ thresholdConds = do+ expr <- numericExprsWithTerms (synthConfig cfg) df+ threshold <- numericThresholds cfg df expr+ condsFromExpr expr threshold +missingnessConditions :: DataFrame -> [Expr Bool]+missingnessConditions df = concatMap missingCond (columnNames df)+ where+ missingCond name = case unsafeGetColumn name df of+ BoxedColumn (Just _) (_ :: V.Vector b) -> [F.isNothing (Col @(Maybe b) name)]+ UnboxedColumn (Just _) (_ :: VU.Vector b) -> [F.isNothing (Col @(Maybe b) name)]+ PackedText (Just _) _ -> [F.isNothing (Col @(Maybe T.Text) name)]+ _ -> []++-- | Thresholds for nullable expressions come from the observed values only. numericThresholds :: TreeConfig -> DataFrame -> NumExpr -> [Double] numericThresholds cfg df (NDouble e) = thresholdsForExpr cfg df e-numericThresholds cfg df (NMaybeDouble e) = thresholdsForExpr cfg df (F.fromMaybe 0 e)+numericThresholds cfg df (NMaybeDouble e) =+ maybe+ []+ (percentilesOf (percentiles cfg) . catMaybes . V.toList)+ (interpretMaybeDoubleCol df e) thresholdsForExpr :: TreeConfig -> DataFrame -> Expr Double -> [Double] thresholdsForExpr cfg df e =@@ -142,7 +161,9 @@ Byte-identical to materializing 'numericConditions' one at a time. -} numericCondVecs :: TreeConfig -> DataFrame -> DataFrame -> [CondVec]-numericCondVecs cfg dfGen df = concatMap forExpr (numericExprsWithTerms (synthConfig cfg) dfGen)+numericCondVecs cfg dfGen df =+ concatMap forExpr (numericExprsWithTerms (synthConfig cfg) dfGen)+ ++ mapMaybe (materializeCondVec df) (missingnessConditions dfGen) where forExpr (NDouble e) = maybe [] (condsForDouble cfg e) (interpretDoubleCol df e) forExpr (NMaybeDouble e) = maybe [] (condsForMaybe cfg e) (interpretMaybeDoubleCol df e)@@ -166,7 +187,7 @@ TreeConfig -> Expr (Maybe Double) -> V.Vector (Maybe Double) -> [CondVec] condsForMaybe cfg e mvals = concatMap (maybeCondsAt e mvals (V.length mvals)) ts where- ts = percentilesOf (percentiles cfg) (map (fromMaybe 0) (V.toList mvals))+ ts = percentilesOf (percentiles cfg) (catMaybes (V.toList mvals)) maybeCondsAt :: Expr (Maybe Double) -> V.Vector (Maybe Double) -> Int -> Double -> [CondVec]
tests-internal/DecisionTree.hs view
@@ -28,7 +28,9 @@ import DataFrame.DecisionTree.Numeric ( NumExpr (NMaybeDouble), generateNumericConds,+ missingnessConditions, numericCols,+ numericCondVecs, numericExprsWithTerms, ) import DataFrame.DecisionTree.Predict (@@ -45,16 +47,22 @@ import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.Expression (Expr (..), eqExpr, getColumns) import DataFrame.Internal.Interpreter (interpret)+import DataFrame.Internal.PackedText (mkPackedContiguous) import qualified DataFrame.LinearSolver import DataFrame.Operators import qualified DataFrameApi as D +import Control.Monad (zipWithM_)+import qualified Data.ByteString as B import Data.Function (on) import Data.List (maximumBy, sort) import qualified Data.Map.Strict as M import qualified Data.Text as T+import qualified Data.Text.Array as A+import Data.Text.Encoding (encodeUtf8) import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU+import Data.Word (Word8) import Test.HUnit ------------------------------------------------------------------------@@ -359,10 +367,6 @@ "dead-branch tree must produce a valid loss in [0,1]" (finalLoss >= 0.0 && finalLoss <= 1.0) ---------------------------------------------------------------------------- Shared fixtures: 4x4 grid-------------------------------------------------------------------------- gridPairs :: [(Double, Double)] gridPairs = [(x, y) | y <- [1 .. 4], x <- [1 .. 4]] @@ -373,10 +377,6 @@ , ("y", DI.fromList (map snd gridPairs)) ] ---------------------------------------------------------------------------- Oblique recovery tests-------------------------------------------------------------------------- taoRecoversSingleObliqueDerived :: Test taoRecoversSingleObliqueDerived = TestCase $ do let labelExpr =@@ -624,7 +624,101 @@ "combined exprs include NMaybeDouble (nullable arithmetic)" (any (\case NMaybeDouble _ -> True; _ -> False) exprs) --- probsFromIndices: counts correct on a 3-row slice+missingnessCondsTest :: Test+missingnessCondsTest = TestCase $ do+ let conds = missingnessConditions (D.exclude ["label"] nullsMixedDF)+ assertEqual "one nullable column -> one missingness cond" 1 (length conds)+ assertBool+ "cond is isNothing x"+ (eqExpr (head conds) (F.isNothing (F.col @(Maybe Double) "x")))+ assertBool+ "no missingness conds for a non-nullable DataFrame"+ (null (missingnessConditions fixtureDF))++poolContainsMissingnessTest :: Test+poolContainsMissingnessTest =+ TestCase $+ assertBool+ "generateNumericConds contains isNothing x"+ ( any+ (`eqExpr` F.isNothing (F.col @(Maybe Double) "x"))+ (generateNumericConds defaultTreeConfig (D.exclude ["label"] nullsMixedDF))+ )++missingnessCondVecTest :: Test+missingnessCondVecTest = TestCase $ do+ let cvs =+ numericCondVecs+ defaultTreeConfig+ (D.exclude ["label"] nullsMixedDF)+ nullsMixedDF+ isMissingCV cv = eqExpr (cvExpr cv) (F.isNothing (F.col @(Maybe Double) "x"))+ expected = VU.fromList [False, True, False, False, True, False]+ case filter isMissingCV cvs of+ (cv : _) -> assertEqual "isNothing vector marks null slots" expected (cvVec cv)+ [] -> assertFailure "no isNothing CondVec in pool"++observedOnlyThresholdsTest :: Test+observedOnlyThresholdsTest = TestCase $ do+ let df =+ D.fromNamedColumns+ [+ ( "x"+ , DI.fromVector+ ( V.fromList+ (replicate 5 Nothing ++ map Just [10, 20, 30, 40, 50]) ::+ V.Vector (Maybe Double)+ )+ )+ ]+ conds = generateNumericConds defaultTreeConfig{percentiles = [50]} df+ leqAt t = F.fromMaybe False (F.col @(Maybe Double) "x" .<= F.lit (t :: Double))+ assertBool+ "median threshold from observed values (30)"+ (any (`eqExpr` leqAt 30) conds)+ assertBool "no null-skewed threshold (10)" (not (any (`eqExpr` leqAt 10) conds))++packedFromTexts :: Maybe [Int] -> [T.Text] -> DI.Column+packedFromTexts nullIdxs ts =+ DI.PackedText bm (mkPackedContiguous arr (VU.fromList offs))+ where+ bytess = map (B.unpack . encodeUtf8) ts+ offs = scanl (+) 0 (map length bytess)+ arr = arrayFromBytes (concat bytess)+ bm = DI.buildBitmapFromNulls (length ts) <$> nullIdxs++arrayFromBytes :: [Word8] -> A.Array+arrayFromBytes ws = A.run $ do+ m <- A.new (length ws)+ zipWithM_ (A.unsafeWrite m) [0 ..] ws+ pure m++-- G5: nullable PackedText columns (what CSV ingest emits for nullable text)+-- get a missingness candidate; non-nullable PackedText does not.+packedMissingnessTest :: Test+packedMissingnessTest = TestCase $ do+ let df =+ D.fromNamedColumns+ [("s", packedFromTexts (Just [1, 3]) ["yes", "", "no", ""])]+ isMissingS = F.isNothing (F.col @(Maybe T.Text) "s")+ conds = missingnessConditions df+ assertEqual "one missingness cond for nullable PackedText" 1 (length conds)+ assertBool "cond is isNothing s" (eqExpr (head conds) isMissingS)+ assertBool+ "no missingness cond for non-nullable PackedText"+ ( null+ ( missingnessConditions+ (D.fromNamedColumns [("t", packedFromTexts Nothing ["a", "b"])])+ )+ )+ case filter (eqExpr isMissingS . cvExpr) (numericCondVecs defaultTreeConfig df df) of+ (cv : _) ->+ assertEqual+ "isNothing vector marks null slots"+ (VU.fromList [False, True, False, True])+ (cvVec cv)+ [] -> assertFailure "no isNothing CondVec for PackedText column"+ probsFromIndicesBasic :: Test probsFromIndicesBasic = TestCase $ do let df =@@ -824,9 +918,6 @@ ("loss must be non-increasing across iterations (got " ++ show losses ++ ")") (all (\(a, b) -> b <= a + 1e-9) pairs) --- C6: When the discrete pool contains an exact-zero-error split (axis-aligned--- works perfectly), the competition picks the simpler discrete candidate--- rather than a similarly-good but more complex linear one. taoLinearVsDiscreteCompetition :: Test taoLinearVsDiscreteCompetition = TestCase $ do let indices = V.enumFromN 0 20@@ -1301,6 +1392,11 @@ , TestLabel "nullableFitZeroLoss" nullableFitZeroLossTest , TestLabel "nullableFitWithNullsNoCrash" nullableFitWithNullsNoCrashTest , TestLabel "numericExprsWithTermsMixed" numericExprsWithTermsMixedTest+ , TestLabel "G1 missingnessConds" missingnessCondsTest+ , TestLabel "G2 poolContainsMissingness" poolContainsMissingnessTest+ , TestLabel "G3 missingnessCondVec" missingnessCondVecTest+ , TestLabel "G4 observedOnlyThresholds" observedOnlyThresholdsTest+ , TestLabel "G5 packedMissingness" packedMissingnessTest , TestLabel "probsFromIndicesBasic" probsFromIndicesBasic , TestLabel "probsFromIndicesSubset" probsFromIndicesSubset , TestLabel "probsFromIndicesSingleClass" probsFromIndicesSingleClass
tests-internal/LinearSolver.hs view
@@ -17,10 +17,6 @@ import System.Random (mkStdGen, randomR) import Test.HUnit ---------------------------------------------------------------------------- Test fixtures and helpers-------------------------------------------------------------------------- -- Generate n points with d features, each value uniform in [-1, 1], from a seed. syntheticPoints :: Int -> Int -> Int -> V.Vector (VU.Vector Double) syntheticPoints seed n d =@@ -88,10 +84,6 @@ else (-margin) + log (1 + exp margin) in sum [loss i | i <- [0 .. n - 1]] / fromIntegral n ---------------------------------------------------------------------------- A1: Recover known hyperplane with no L1-------------------------------------------------------------------------- testA1RecoverHyperplane :: Test testA1RecoverHyperplane = TestCase $ do let groundTruth = VU.fromList [0.7, -0.5]@@ -116,10 +108,6 @@ (cosSim > 0.99) assertBool "all training points predicted correctly" sameSignAll ---------------------------------------------------------------------------- A2: L1 produces sparse weights-------------------------------------------------------------------------- testA2L1Sparsity :: Test testA2L1Sparsity = TestCase $ do let groundTruth = VU.fromList [0, 1.2, 0, 0, -1.5, 0, 0, 0, 0, 0]@@ -161,10 +149,6 @@ ) (noiseZero >= 6) ---------------------------------------------------------------------------- A3: Convergence on well-conditioned input-------------------------------------------------------------------------- testA3Convergence :: Test testA3Convergence = TestCase $ do let groundTruth = VU.fromList [1.0, -0.5, 0.7]@@ -192,10 +176,6 @@ ) (lossFit < loss0) ---------------------------------------------------------------------------- A4: Final loss <= initial loss (monotone or near-monotone in FISTA)-------------------------------------------------------------------------- testA4LossNotIncreasing :: Test testA4LossNotIncreasing = TestCase $ do let groundTruth = VU.fromList [0.8, 0.4]@@ -214,10 +194,6 @@ ) (lossFit <= loss0 + 1e-9) ---------------------------------------------------------------------------- A5: Degenerate input — all labels +1-------------------------------------------------------------------------- testA5AllSameDirection :: Test testA5AllSameDirection = TestCase $ do let rows = syntheticPoints 4 50 3@@ -235,10 +211,6 @@ "all-same labels should produce a positive-predicting model" allPositive ---------------------------------------------------------------------------- A6: Degenerate — empty input-------------------------------------------------------------------------- testA6Empty :: Test testA6Empty = TestCase $ do let cfg = defaultSolverConfig@@ -252,10 +224,6 @@ (lmWeights model) assertEqual "empty input -> zero intercept" 0 (lmIntercept model) ---------------------------------------------------------------------------- A7: Degenerate — constant feature-------------------------------------------------------------------------- testA7ConstantFeature :: Test testA7ConstantFeature = TestCase $ do let baseRows = syntheticPoints 5 100 1@@ -286,10 +254,6 @@ (ws !! 1 /= 0) assertBool "no NaN/Inf" (not anyBad) ---------------------------------------------------------------------------- A8: Numerical stability with large feature values-------------------------------------------------------------------------- testA8LargeValues :: Test testA8LargeValues = TestCase $ do let scale = 1000.0 :: Double@@ -314,11 +278,6 @@ ) (sameSigns >= 90) ---------------------------------------------------------------------------- A9: Standardization round-trip — recovered weights point in the true--- direction even when raw-feature scales differ by orders of magnitude.-------------------------------------------------------------------------- testA9StandardizationRoundTrip :: Test testA9StandardizationRoundTrip = TestCase $ do let nRows = 80 :: Int@@ -356,10 +315,6 @@ ) (cs > 0.95) ---------------------------------------------------------------------------- A10: Determinism — same input -> same output-------------------------------------------------------------------------- testA10Determinism :: Test testA10Determinism = TestCase $ do let groundTruth = VU.fromList [0.6, 0.4]@@ -371,10 +326,6 @@ assertEqual "same input -> same weights" (lmWeights m1) (lmWeights m2) assertEqual "same input -> same intercept" (lmIntercept m1) (lmIntercept m2) ---------------------------------------------------------------------------- A11: Two-feature ground truth recovery (w_2/w_1 ratio)-------------------------------------------------------------------------- testA11GroundTruthRatio :: Test testA11GroundTruthRatio = TestCase $ do let groundTruth = VU.fromList [1.0, 2.0]@@ -402,10 +353,6 @@ ("b/w1 should approximate -3.0 (got " ++ show biasRatio ++ ")") (biasRatio > -3.4 && biasRatio < -2.6) ---------------------------------------------------------------------------- B1: modelToExpr produces a well-typed Expr Bool-------------------------------------------------------------------------- testB1ExprWellTyped :: Test testB1ExprWellTyped = TestCase $ do let model =@@ -432,10 +379,6 @@ Right vals -> assertEqual "Expr matches manual evaluation" manual (V.toList vals) ---------------------------------------------------------------------------- B2: Zero weights are dropped from the resulting Expr-------------------------------------------------------------------------- testB2ZeroWeightsPruned :: Test testB2ZeroWeightsPruned = TestCase $ do let model =@@ -448,11 +391,6 @@ cols = sort (getColumns expr) assertEqual "only column b appears in the Expr" ["b"] cols ---------------------------------------------------------------------------- A14: Constant feature at large raw value — weight must be exactly 0--- and no NaN/Inf leaks into the rest of the fit.-------------------------------------------------------------------------- testA14ConstantHugeValue :: Test testA14ConstantHugeValue = TestCase $ do let baseRows = syntheticPoints 14 100 1@@ -478,10 +416,6 @@ ("signal feature has non-zero weight (got " ++ show (ws !! 1) ++ ")") (ws !! 1 /= 0) ---------------------------------------------------------------------------- A15: Variance exactly zero (all rows identical for that column).-------------------------------------------------------------------------- testA15AllZeroFeature :: Test testA15AllZeroFeature = TestCase $ do let baseRows = syntheticPoints 15 80 1@@ -499,11 +433,6 @@ assertEqual "zero-variance column has weight zero" 0 w0 assertBool ("signal weight non-zero (" ++ show (ws !! 1) ++ ")") (ws !! 1 /= 0) ---------------------------------------------------------------------------- A16: Severely imbalanced labels (99:1) — should not collapse to a--- constant predictor on the majority class without some learning.-------------------------------------------------------------------------- testA16ImbalancedLabels :: Test testA16ImbalancedLabels = TestCase $ do let nPos = 99@@ -521,10 +450,6 @@ assertBool "no NaN/Inf with 99:1 imbalance" (not anyBad) assertBool ("intercept favors majority class (got b=" ++ show b ++ ")") (b > 0) ---------------------------------------------------------------------------- A17: Mixed per-feature raw scales — should not diverge.-------------------------------------------------------------------------- testA17ImbalancedRawScales :: Test testA17ImbalancedRawScales = TestCase $ do let baseRows = syntheticPoints 17 100 3@@ -551,10 +476,6 @@ ("non-divergent under wild scales (got " ++ show correct ++ "/100)") (correct >= 65) ---------------------------------------------------------------------------- A12: maxIter = 0 returns the initial point unchanged-------------------------------------------------------------------------- testA12MaxIterZero :: Test testA12MaxIterZero = TestCase $ do let rows = syntheticPoints 20 50 2@@ -567,11 +488,6 @@ (lmWeights model) assertEqual "maxIter=0 returns zero intercept" 0 (lmIntercept model) ---------------------------------------------------------------------------- A13: maxIter = 1 takes exactly one prox step (results differ from--- the initial zero point but may not be near the optimum).-------------------------------------------------------------------------- testA13MaxIterOne :: Test testA13MaxIterOne = TestCase $ do let rows = syntheticPoints 21 80 2@@ -589,12 +505,6 @@ badB = isNaN (lmIntercept m1) || isInfinite (lmIntercept m1) assertBool "no NaN/Inf after one iteration" (not (badW || badB)) ---------------------------------------------------------------------------- PR 3: Elastic Net recovery on correlated-feature pairs. Pure L1 picks one--- of two correlated informative features; Elastic Net keeps both non-zero--- (Zou & Hastie 2005 grouping effect). Cases: ρ ≈ 0.97 and ρ ≈ 0.7.-------------------------------------------------------------------------- -- Generate two correlated features f0, f1 with correlation ρ, plus -- noise features f2..f7. Truth is sign(f0 + f1). correlatedPairData ::@@ -677,12 +587,6 @@ ("ρ=0.7 EN grouping: |w0/w1| ∈ [0.33, 3.0]; got ratio=" ++ show ratio) (ratio >= 0.33 && ratio <= 3.0) ---------------------------------------------------------------------------- PR 3: A20 — class-balanced fit on 95/5 imbalance. Unweighted, the intercept--- polarises toward logit(0.95) ≈ 2.94; with class-balanced weights it sits--- near 0 and predictions become roughly balanced on a symmetric test set.-------------------------------------------------------------------------- testA20ClassBalancedFit :: Test testA20ClassBalancedFit = TestCase $ do let n = 200 :: Int@@ -749,10 +653,6 @@ ++ show fracBal ) (fracBal >= 0.40 && fracBal <= 0.60)----------------------------------------------------------------------------- Test list------------------------------------------------------------------------- tests :: [Test] tests =