diff --git a/dataframe-learn.cabal b/dataframe-learn.cabal --- a/dataframe-learn.cabal +++ b/dataframe-learn.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe-learn -version: 2.3.0.0 +version: 2.4.0.0 synopsis: Interpretable, expression-returning machine learning for the dataframe ecosystem. description: A small scikit-learn-style ML library where every model returns both an diff --git a/src-internal/DataFrame/DecisionTree/Numeric.hs b/src-internal/DataFrame/DecisionTree/Numeric.hs --- a/src-internal/DataFrame/DecisionTree/Numeric.hs +++ b/src-internal/DataFrame/DecisionTree/Numeric.hs @@ -6,8 +6,10 @@ {-# LANGUAGE TypeApplications #-} {- | Numeric split candidates: per-column Double expressions, arithmetic -expansion, and threshold conditions. 'numericCondVecs' materializes the -pool with one interpret per distinct expression. +expansion, and threshold conditions. Nullable columns also contribute an +@isNothing@ candidate, so missingness is splittable directly rather than +only via the null-routing of value splits. 'numericCondVecs' materializes +the pool with one interpret per distinct expression. -} module DataFrame.DecisionTree.Numeric ( NumExpr (..), @@ -16,13 +18,14 @@ combineNumExprs, numericConditions, generateNumericConds, + missingnessConditions, percentilesOf, numericCondVecs, numericExprsWithTerms, numericCols, ) where -import DataFrame.DecisionTree.CondVec (CondVec (..)) +import DataFrame.DecisionTree.CondVec (CondVec (..), materializeCondVec) import DataFrame.DecisionTree.Types (SynthConfig (..), TreeConfig (..)) import qualified DataFrame.Functions as F import DataFrame.Internal.Column @@ -33,7 +36,7 @@ import DataFrame.Operators import Data.List (sort) -import Data.Maybe (fromMaybe) +import Data.Maybe (catMaybes, mapMaybe) import qualified Data.Set as Set import qualified Data.Text as T import Data.Type.Equality (testEquality, (:~:) (..)) @@ -95,14 +98,30 @@ numericConditions = generateNumericConds generateNumericConds :: TreeConfig -> DataFrame -> [Expr Bool] -generateNumericConds cfg df = do - expr <- numericExprsWithTerms (synthConfig cfg) df - threshold <- numericThresholds cfg df expr - condsFromExpr expr threshold +generateNumericConds cfg df = thresholdConds ++ missingnessConditions df + where + thresholdConds = do + expr <- numericExprsWithTerms (synthConfig cfg) df + threshold <- numericThresholds cfg df expr + condsFromExpr expr threshold +missingnessConditions :: DataFrame -> [Expr Bool] +missingnessConditions df = concatMap missingCond (columnNames df) + where + missingCond name = case unsafeGetColumn name df of + BoxedColumn (Just _) (_ :: V.Vector b) -> [F.isNothing (Col @(Maybe b) name)] + UnboxedColumn (Just _) (_ :: VU.Vector b) -> [F.isNothing (Col @(Maybe b) name)] + PackedText (Just _) _ -> [F.isNothing (Col @(Maybe T.Text) name)] + _ -> [] + +-- | Thresholds for nullable expressions come from the observed values only. numericThresholds :: TreeConfig -> DataFrame -> NumExpr -> [Double] numericThresholds cfg df (NDouble e) = thresholdsForExpr cfg df e -numericThresholds cfg df (NMaybeDouble e) = thresholdsForExpr cfg df (F.fromMaybe 0 e) +numericThresholds cfg df (NMaybeDouble e) = + maybe + [] + (percentilesOf (percentiles cfg) . catMaybes . V.toList) + (interpretMaybeDoubleCol df e) thresholdsForExpr :: TreeConfig -> DataFrame -> Expr Double -> [Double] thresholdsForExpr cfg df e = @@ -142,7 +161,9 @@ Byte-identical to materializing 'numericConditions' one at a time. -} numericCondVecs :: TreeConfig -> DataFrame -> DataFrame -> [CondVec] -numericCondVecs cfg dfGen df = concatMap forExpr (numericExprsWithTerms (synthConfig cfg) dfGen) +numericCondVecs cfg dfGen df = + concatMap forExpr (numericExprsWithTerms (synthConfig cfg) dfGen) + ++ mapMaybe (materializeCondVec df) (missingnessConditions dfGen) where forExpr (NDouble e) = maybe [] (condsForDouble cfg e) (interpretDoubleCol df e) forExpr (NMaybeDouble e) = maybe [] (condsForMaybe cfg e) (interpretMaybeDoubleCol df e) @@ -166,7 +187,7 @@ TreeConfig -> Expr (Maybe Double) -> V.Vector (Maybe Double) -> [CondVec] condsForMaybe cfg e mvals = concatMap (maybeCondsAt e mvals (V.length mvals)) ts where - ts = percentilesOf (percentiles cfg) (map (fromMaybe 0) (V.toList mvals)) + ts = percentilesOf (percentiles cfg) (catMaybes (V.toList mvals)) maybeCondsAt :: Expr (Maybe Double) -> V.Vector (Maybe Double) -> Int -> Double -> [CondVec] diff --git a/tests-internal/DecisionTree.hs b/tests-internal/DecisionTree.hs --- a/tests-internal/DecisionTree.hs +++ b/tests-internal/DecisionTree.hs @@ -28,7 +28,9 @@ import DataFrame.DecisionTree.Numeric ( NumExpr (NMaybeDouble), generateNumericConds, + missingnessConditions, numericCols, + numericCondVecs, numericExprsWithTerms, ) import DataFrame.DecisionTree.Predict ( @@ -45,16 +47,22 @@ import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.Expression (Expr (..), eqExpr, getColumns) import DataFrame.Internal.Interpreter (interpret) +import DataFrame.Internal.PackedText (mkPackedContiguous) import qualified DataFrame.LinearSolver import DataFrame.Operators import qualified DataFrameApi as D +import Control.Monad (zipWithM_) +import qualified Data.ByteString as B import Data.Function (on) import Data.List (maximumBy, sort) import qualified Data.Map.Strict as M import qualified Data.Text as T +import qualified Data.Text.Array as A +import Data.Text.Encoding (encodeUtf8) import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU +import Data.Word (Word8) import Test.HUnit ------------------------------------------------------------------------ @@ -359,10 +367,6 @@ "dead-branch tree must produce a valid loss in [0,1]" (finalLoss >= 0.0 && finalLoss <= 1.0) ------------------------------------------------------------------------- --- Shared fixtures: 4x4 grid ------------------------------------------------------------------------- - gridPairs :: [(Double, Double)] gridPairs = [(x, y) | y <- [1 .. 4], x <- [1 .. 4]] @@ -373,10 +377,6 @@ , ("y", DI.fromList (map snd gridPairs)) ] ------------------------------------------------------------------------- --- Oblique recovery tests ------------------------------------------------------------------------- - taoRecoversSingleObliqueDerived :: Test taoRecoversSingleObliqueDerived = TestCase $ do let labelExpr = @@ -624,7 +624,101 @@ "combined exprs include NMaybeDouble (nullable arithmetic)" (any (\case NMaybeDouble _ -> True; _ -> False) exprs) --- probsFromIndices: counts correct on a 3-row slice +missingnessCondsTest :: Test +missingnessCondsTest = TestCase $ do + let conds = missingnessConditions (D.exclude ["label"] nullsMixedDF) + assertEqual "one nullable column -> one missingness cond" 1 (length conds) + assertBool + "cond is isNothing x" + (eqExpr (head conds) (F.isNothing (F.col @(Maybe Double) "x"))) + assertBool + "no missingness conds for a non-nullable DataFrame" + (null (missingnessConditions fixtureDF)) + +poolContainsMissingnessTest :: Test +poolContainsMissingnessTest = + TestCase $ + assertBool + "generateNumericConds contains isNothing x" + ( any + (`eqExpr` F.isNothing (F.col @(Maybe Double) "x")) + (generateNumericConds defaultTreeConfig (D.exclude ["label"] nullsMixedDF)) + ) + +missingnessCondVecTest :: Test +missingnessCondVecTest = TestCase $ do + let cvs = + numericCondVecs + defaultTreeConfig + (D.exclude ["label"] nullsMixedDF) + nullsMixedDF + isMissingCV cv = eqExpr (cvExpr cv) (F.isNothing (F.col @(Maybe Double) "x")) + expected = VU.fromList [False, True, False, False, True, False] + case filter isMissingCV cvs of + (cv : _) -> assertEqual "isNothing vector marks null slots" expected (cvVec cv) + [] -> assertFailure "no isNothing CondVec in pool" + +observedOnlyThresholdsTest :: Test +observedOnlyThresholdsTest = TestCase $ do + let df = + D.fromNamedColumns + [ + ( "x" + , DI.fromVector + ( V.fromList + (replicate 5 Nothing ++ map Just [10, 20, 30, 40, 50]) :: + V.Vector (Maybe Double) + ) + ) + ] + conds = generateNumericConds defaultTreeConfig{percentiles = [50]} df + leqAt t = F.fromMaybe False (F.col @(Maybe Double) "x" .<= F.lit (t :: Double)) + assertBool + "median threshold from observed values (30)" + (any (`eqExpr` leqAt 30) conds) + assertBool "no null-skewed threshold (10)" (not (any (`eqExpr` leqAt 10) conds)) + +packedFromTexts :: Maybe [Int] -> [T.Text] -> DI.Column +packedFromTexts nullIdxs ts = + DI.PackedText bm (mkPackedContiguous arr (VU.fromList offs)) + where + bytess = map (B.unpack . encodeUtf8) ts + offs = scanl (+) 0 (map length bytess) + arr = arrayFromBytes (concat bytess) + bm = DI.buildBitmapFromNulls (length ts) <$> nullIdxs + +arrayFromBytes :: [Word8] -> A.Array +arrayFromBytes ws = A.run $ do + m <- A.new (length ws) + zipWithM_ (A.unsafeWrite m) [0 ..] ws + pure m + +-- G5: nullable PackedText columns (what CSV ingest emits for nullable text) +-- get a missingness candidate; non-nullable PackedText does not. +packedMissingnessTest :: Test +packedMissingnessTest = TestCase $ do + let df = + D.fromNamedColumns + [("s", packedFromTexts (Just [1, 3]) ["yes", "", "no", ""])] + isMissingS = F.isNothing (F.col @(Maybe T.Text) "s") + conds = missingnessConditions df + assertEqual "one missingness cond for nullable PackedText" 1 (length conds) + assertBool "cond is isNothing s" (eqExpr (head conds) isMissingS) + assertBool + "no missingness cond for non-nullable PackedText" + ( null + ( missingnessConditions + (D.fromNamedColumns [("t", packedFromTexts Nothing ["a", "b"])]) + ) + ) + case filter (eqExpr isMissingS . cvExpr) (numericCondVecs defaultTreeConfig df df) of + (cv : _) -> + assertEqual + "isNothing vector marks null slots" + (VU.fromList [False, True, False, True]) + (cvVec cv) + [] -> assertFailure "no isNothing CondVec for PackedText column" + probsFromIndicesBasic :: Test probsFromIndicesBasic = TestCase $ do let df = @@ -824,9 +918,6 @@ ("loss must be non-increasing across iterations (got " ++ show losses ++ ")") (all (\(a, b) -> b <= a + 1e-9) pairs) --- C6: When the discrete pool contains an exact-zero-error split (axis-aligned --- works perfectly), the competition picks the simpler discrete candidate --- rather than a similarly-good but more complex linear one. taoLinearVsDiscreteCompetition :: Test taoLinearVsDiscreteCompetition = TestCase $ do let indices = V.enumFromN 0 20 @@ -1301,6 +1392,11 @@ , TestLabel "nullableFitZeroLoss" nullableFitZeroLossTest , TestLabel "nullableFitWithNullsNoCrash" nullableFitWithNullsNoCrashTest , TestLabel "numericExprsWithTermsMixed" numericExprsWithTermsMixedTest + , TestLabel "G1 missingnessConds" missingnessCondsTest + , TestLabel "G2 poolContainsMissingness" poolContainsMissingnessTest + , TestLabel "G3 missingnessCondVec" missingnessCondVecTest + , TestLabel "G4 observedOnlyThresholds" observedOnlyThresholdsTest + , TestLabel "G5 packedMissingness" packedMissingnessTest , TestLabel "probsFromIndicesBasic" probsFromIndicesBasic , TestLabel "probsFromIndicesSubset" probsFromIndicesSubset , TestLabel "probsFromIndicesSingleClass" probsFromIndicesSingleClass diff --git a/tests-internal/LinearSolver.hs b/tests-internal/LinearSolver.hs --- a/tests-internal/LinearSolver.hs +++ b/tests-internal/LinearSolver.hs @@ -17,10 +17,6 @@ import System.Random (mkStdGen, randomR) import Test.HUnit ------------------------------------------------------------------------- --- Test fixtures and helpers ------------------------------------------------------------------------- - -- Generate n points with d features, each value uniform in [-1, 1], from a seed. syntheticPoints :: Int -> Int -> Int -> V.Vector (VU.Vector Double) syntheticPoints seed n d = @@ -88,10 +84,6 @@ else (-margin) + log (1 + exp margin) in sum [loss i | i <- [0 .. n - 1]] / fromIntegral n ------------------------------------------------------------------------- --- A1: Recover known hyperplane with no L1 ------------------------------------------------------------------------- - testA1RecoverHyperplane :: Test testA1RecoverHyperplane = TestCase $ do let groundTruth = VU.fromList [0.7, -0.5] @@ -116,10 +108,6 @@ (cosSim > 0.99) assertBool "all training points predicted correctly" sameSignAll ------------------------------------------------------------------------- --- A2: L1 produces sparse weights ------------------------------------------------------------------------- - testA2L1Sparsity :: Test testA2L1Sparsity = TestCase $ do let groundTruth = VU.fromList [0, 1.2, 0, 0, -1.5, 0, 0, 0, 0, 0] @@ -161,10 +149,6 @@ ) (noiseZero >= 6) ------------------------------------------------------------------------- --- A3: Convergence on well-conditioned input ------------------------------------------------------------------------- - testA3Convergence :: Test testA3Convergence = TestCase $ do let groundTruth = VU.fromList [1.0, -0.5, 0.7] @@ -192,10 +176,6 @@ ) (lossFit < loss0) ------------------------------------------------------------------------- --- A4: Final loss <= initial loss (monotone or near-monotone in FISTA) ------------------------------------------------------------------------- - testA4LossNotIncreasing :: Test testA4LossNotIncreasing = TestCase $ do let groundTruth = VU.fromList [0.8, 0.4] @@ -214,10 +194,6 @@ ) (lossFit <= loss0 + 1e-9) ------------------------------------------------------------------------- --- A5: Degenerate input — all labels +1 ------------------------------------------------------------------------- - testA5AllSameDirection :: Test testA5AllSameDirection = TestCase $ do let rows = syntheticPoints 4 50 3 @@ -235,10 +211,6 @@ "all-same labels should produce a positive-predicting model" allPositive ------------------------------------------------------------------------- --- A6: Degenerate — empty input ------------------------------------------------------------------------- - testA6Empty :: Test testA6Empty = TestCase $ do let cfg = defaultSolverConfig @@ -252,10 +224,6 @@ (lmWeights model) assertEqual "empty input -> zero intercept" 0 (lmIntercept model) ------------------------------------------------------------------------- --- A7: Degenerate — constant feature ------------------------------------------------------------------------- - testA7ConstantFeature :: Test testA7ConstantFeature = TestCase $ do let baseRows = syntheticPoints 5 100 1 @@ -286,10 +254,6 @@ (ws !! 1 /= 0) assertBool "no NaN/Inf" (not anyBad) ------------------------------------------------------------------------- --- A8: Numerical stability with large feature values ------------------------------------------------------------------------- - testA8LargeValues :: Test testA8LargeValues = TestCase $ do let scale = 1000.0 :: Double @@ -314,11 +278,6 @@ ) (sameSigns >= 90) ------------------------------------------------------------------------- --- A9: Standardization round-trip — recovered weights point in the true --- direction even when raw-feature scales differ by orders of magnitude. ------------------------------------------------------------------------- - testA9StandardizationRoundTrip :: Test testA9StandardizationRoundTrip = TestCase $ do let nRows = 80 :: Int @@ -356,10 +315,6 @@ ) (cs > 0.95) ------------------------------------------------------------------------- --- A10: Determinism — same input -> same output ------------------------------------------------------------------------- - testA10Determinism :: Test testA10Determinism = TestCase $ do let groundTruth = VU.fromList [0.6, 0.4] @@ -371,10 +326,6 @@ assertEqual "same input -> same weights" (lmWeights m1) (lmWeights m2) assertEqual "same input -> same intercept" (lmIntercept m1) (lmIntercept m2) ------------------------------------------------------------------------- --- A11: Two-feature ground truth recovery (w_2/w_1 ratio) ------------------------------------------------------------------------- - testA11GroundTruthRatio :: Test testA11GroundTruthRatio = TestCase $ do let groundTruth = VU.fromList [1.0, 2.0] @@ -402,10 +353,6 @@ ("b/w1 should approximate -3.0 (got " ++ show biasRatio ++ ")") (biasRatio > -3.4 && biasRatio < -2.6) ------------------------------------------------------------------------- --- B1: modelToExpr produces a well-typed Expr Bool ------------------------------------------------------------------------- - testB1ExprWellTyped :: Test testB1ExprWellTyped = TestCase $ do let model = @@ -432,10 +379,6 @@ Right vals -> assertEqual "Expr matches manual evaluation" manual (V.toList vals) ------------------------------------------------------------------------- --- B2: Zero weights are dropped from the resulting Expr ------------------------------------------------------------------------- - testB2ZeroWeightsPruned :: Test testB2ZeroWeightsPruned = TestCase $ do let model = @@ -448,11 +391,6 @@ cols = sort (getColumns expr) assertEqual "only column b appears in the Expr" ["b"] cols ------------------------------------------------------------------------- --- A14: Constant feature at large raw value — weight must be exactly 0 --- and no NaN/Inf leaks into the rest of the fit. ------------------------------------------------------------------------- - testA14ConstantHugeValue :: Test testA14ConstantHugeValue = TestCase $ do let baseRows = syntheticPoints 14 100 1 @@ -478,10 +416,6 @@ ("signal feature has non-zero weight (got " ++ show (ws !! 1) ++ ")") (ws !! 1 /= 0) ------------------------------------------------------------------------- --- A15: Variance exactly zero (all rows identical for that column). ------------------------------------------------------------------------- - testA15AllZeroFeature :: Test testA15AllZeroFeature = TestCase $ do let baseRows = syntheticPoints 15 80 1 @@ -499,11 +433,6 @@ assertEqual "zero-variance column has weight zero" 0 w0 assertBool ("signal weight non-zero (" ++ show (ws !! 1) ++ ")") (ws !! 1 /= 0) ------------------------------------------------------------------------- --- A16: Severely imbalanced labels (99:1) — should not collapse to a --- constant predictor on the majority class without some learning. ------------------------------------------------------------------------- - testA16ImbalancedLabels :: Test testA16ImbalancedLabels = TestCase $ do let nPos = 99 @@ -521,10 +450,6 @@ assertBool "no NaN/Inf with 99:1 imbalance" (not anyBad) assertBool ("intercept favors majority class (got b=" ++ show b ++ ")") (b > 0) ------------------------------------------------------------------------- --- A17: Mixed per-feature raw scales — should not diverge. ------------------------------------------------------------------------- - testA17ImbalancedRawScales :: Test testA17ImbalancedRawScales = TestCase $ do let baseRows = syntheticPoints 17 100 3 @@ -551,10 +476,6 @@ ("non-divergent under wild scales (got " ++ show correct ++ "/100)") (correct >= 65) ------------------------------------------------------------------------- --- A12: maxIter = 0 returns the initial point unchanged ------------------------------------------------------------------------- - testA12MaxIterZero :: Test testA12MaxIterZero = TestCase $ do let rows = syntheticPoints 20 50 2 @@ -567,11 +488,6 @@ (lmWeights model) assertEqual "maxIter=0 returns zero intercept" 0 (lmIntercept model) ------------------------------------------------------------------------- --- A13: maxIter = 1 takes exactly one prox step (results differ from --- the initial zero point but may not be near the optimum). ------------------------------------------------------------------------- - testA13MaxIterOne :: Test testA13MaxIterOne = TestCase $ do let rows = syntheticPoints 21 80 2 @@ -589,12 +505,6 @@ badB = isNaN (lmIntercept m1) || isInfinite (lmIntercept m1) assertBool "no NaN/Inf after one iteration" (not (badW || badB)) ------------------------------------------------------------------------- --- PR 3: Elastic Net recovery on correlated-feature pairs. Pure L1 picks one --- of two correlated informative features; Elastic Net keeps both non-zero --- (Zou & Hastie 2005 grouping effect). Cases: ρ ≈ 0.97 and ρ ≈ 0.7. ------------------------------------------------------------------------- - -- Generate two correlated features f0, f1 with correlation ρ, plus -- noise features f2..f7. Truth is sign(f0 + f1). correlatedPairData :: @@ -677,12 +587,6 @@ ("ρ=0.7 EN grouping: |w0/w1| ∈ [0.33, 3.0]; got ratio=" ++ show ratio) (ratio >= 0.33 && ratio <= 3.0) ------------------------------------------------------------------------- --- PR 3: A20 — class-balanced fit on 95/5 imbalance. Unweighted, the intercept --- polarises toward logit(0.95) ≈ 2.94; with class-balanced weights it sits --- near 0 and predictions become roughly balanced on a symmetric test set. ------------------------------------------------------------------------- - testA20ClassBalancedFit :: Test testA20ClassBalancedFit = TestCase $ do let n = 200 :: Int @@ -749,10 +653,6 @@ ++ show fracBal ) (fracBal >= 0.40 && fracBal <= 0.60) - ------------------------------------------------------------------------- --- Test list ------------------------------------------------------------------------- tests :: [Test] tests =