From b149c7629c1157c6aaa10237d3d876e81feeb6bf Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Wed, 7 Oct 2026 02:50:09 -0700 Subject: [PATCH 1/2] feat: GBM score-list leak + oblique TAO splits dropped on Int columns --- .../DataFrame/DecisionTree/Linear.hs | 28 +++++-- dataframe-learn/src/DataFrame/Boosting/GBM.hs | 79 ++++++++++++++++--- .../tests-internal/DecisionTree.hs | 24 +++++- 3 files changed, 111 insertions(+), 20 deletions(-) diff --git a/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs b/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs index ff8e962a..426306cd 100644 --- a/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs +++ b/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs @@ -23,6 +23,8 @@ import DataFrame.DecisionTree.Types ( ) import DataFrame.Internal.Column (TypedColumn (..), toVector) import DataFrame.Internal.DataFrame (DataFrame) +import DataFrame.Expression.Operators ((.*.), (.+.), (.>.)) +import qualified DataFrame.Functions as F import DataFrame.Internal.Expression (Expr, getColumns) import DataFrame.Internal.Interpreter (interpret) import qualified DataFrame.LinearSolver as LS @@ -52,23 +54,37 @@ fitLinearCandidate :: fitLinearCandidate cfg target df carePoints = case materializedFeatures target df carePoints of [] -> Nothing - mats -> linearFromFeatures cfg carePoints mats + feats -> linearFromFeatures cfg carePoints feats +{- | Each feature's materialised care values together with the Double expression +that produced them, so the hyperplane can be rebuilt over Int or nullable columns. +-} materializedFeatures :: - T.Text -> DataFrame -> [CarePoint] -> [(T.Text, VU.Vector Double)] + T.Text -> DataFrame -> [CarePoint] -> [(Expr Double, (T.Text, VU.Vector Double))] materializedFeatures target df carePoints = - mapMaybe (materializeFeatureForCare df carePoints) (featureCols target df) + mapMaybe + (\ne -> (\m -> (doubleExprOf ne (snd m), m)) <$> materializeFeatureForCare df carePoints ne) + (featureCols target df) + +-- | The expression behind a feature; a nullable one takes the care-point mean, as its values did. +doubleExprOf :: NumExpr -> VU.Vector Double -> Expr Double +doubleExprOf (NDouble e) _ = e +doubleExprOf (NMaybeDouble e) vals = F.fromMaybe (VU.sum vals / fromIntegral (max 1 (VU.length vals))) e featureCols :: T.Text -> DataFrame -> [NumExpr] featureCols target df = filter (notElem target . numExprCols) (numericCols df) linearFromFeatures :: - TreeConfig -> [CarePoint] -> [(T.Text, VU.Vector Double)] -> Maybe (Expr Bool) -linearFromFeatures cfg carePoints mats + TreeConfig -> [CarePoint] -> [(Expr Double, (T.Text, VU.Vector Double))] -> Maybe (Expr Bool) +linearFromFeatures cfg carePoints feats | VU.all (== 0) weights = Nothing | degenerateHyperplane rows weights (LS.lmIntercept model) = Nothing - | otherwise = Just (LS.modelToExpr model) + | otherwise = Just hyperplane where + mats = map snd feats + hyperplane = + foldl (\acc (w, e) -> acc .+. (F.lit w .*. e)) (F.lit (LS.lmIntercept model)) [(w, e) | (w, (e, _)) <- zip (VU.toList weights) feats, w /= 0] + .>. F.lit (0 :: Double) rows = careRowsFromFeatures (length carePoints) mats labels = careLabels carePoints model = diff --git a/dataframe-learn/src/DataFrame/Boosting/GBM.hs b/dataframe-learn/src/DataFrame/Boosting/GBM.hs index 2227d5fa..907d5a8e 100644 --- a/dataframe-learn/src/DataFrame/Boosting/GBM.hs +++ b/dataframe-learn/src/DataFrame/Boosting/GBM.hs @@ -31,15 +31,17 @@ import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import DataFrame.Errors (DataFrameException (..)) -import DataFrame.DecisionTree.Cart (cartFeatures) +import DataFrame.DecisionTree.Cart (cartFeaturesByColumn) import DataFrame.DecisionTree.Fit (treeToExpr) import DataFrame.DecisionTree.Histogram ( TreeLimits (..), binFeatures, - fitBinnedTree, + fitBinnedTreeOn, ) import DataFrame.DecisionTree.Types (Tree) import DataFrame.Expression.Operators ((.*.), (.+.), (.>.)) +import Data.Maybe (fromMaybe) +import System.Random (StdGen, mkStdGen, randoms) import DataFrame.Featurize.Internal (targetDoubles) import qualified DataFrame.Functions as F import DataFrame.Internal.DataFrame (DataFrame) @@ -60,8 +62,21 @@ data GBConfig = GBConfig distinct values split exactly as on the raw values. -} , gbSeed :: !Int + , gbL2 :: !Double + -- ^ L2 penalty on leaf values (XGBoost's @lambda@). + , gbMinChildWeight :: !Double + -- ^ Smallest Hessian sum a leaf may have (XGBoost's @min_child_weight@). + , gbFeatureGroups :: ![[T.Text]] + -- ^ Tree @m@ splits only on columns of group @m mod (number of groups)@. + -- Empty: every tree may use every column. + , gbBaseScore :: !(Maybe (Expr Double)) + -- ^ Boost from these per-row scores instead of a constant; the prediction adds them back. + , gbSubsample :: !Double + -- ^ Fraction of rows each tree is fitted on (1 = all); drawn per tree from 'gbSeed'. + , gbColsample :: !Double + -- ^ Fraction of the allowed features each tree may split on (1 = all); drawn per tree from 'gbSeed'. } - deriving (Eq, Show) + deriving (Show) defaultGBConfig :: GBConfig defaultGBConfig = @@ -72,6 +87,12 @@ defaultGBConfig = , gbMaxDepth = 3 , gbMaxBins = 1024 , gbSeed = 0 + , gbL2 = 0 + , gbMinChildWeight = 0 + , gbFeatureGroups = [] + , gbBaseScore = Nothing + , gbSubsample = 1 + , gbColsample = 1 } {- | A fitted gradient-boosting model. 'gbInit' is the constant initial score @@ -85,6 +106,7 @@ data GBModel = GBModel , gbModelLoss :: !GBLoss , gbTrainScore :: !(VU.Vector Double) , gbFeatureUsage :: !(M.Map T.Text Int) + , gbBase :: !(Maybe (Expr Double)) } deriving (Show) @@ -106,8 +128,23 @@ fitGBM cfg target@(Col name) df = (gbLoss cfg) (VU.fromList (reverse scores)) usage + (gbBaseScore cfg) where - binned = binFeatures (gbMaxBins cfg) (V.fromList (cartFeatures name df)) + byColumn = cartFeaturesByColumn name df + binned = binFeatures (gbMaxBins cfg) (V.fromList (map snd byColumn)) + allFeatures = VU.enumFromN 0 (length byColumn) + groupFeatures = + [ VU.fromList [j | (j, (c, _)) <- zip [0 ..] byColumn, c `elem` g] + | g <- gbFeatureGroups cfg + ] + featuresFor m = + let allowed = case groupFeatures of + [] -> allFeatures + gs -> gs !! (m `mod` length gs) + in if gbColsample cfg >= 1 then allowed else keepFraction (gbColsample cfg) (mkStdGen (7919 * gbSeed cfg + 2 * m + 1)) allowed + rowMask m + | gbSubsample cfg >= 1 = Nothing + | otherwise = Just (VU.fromList (take n (map (\u -> if u < gbSubsample cfg then 1 else 0) (randoms (mkStdGen (7919 * gbSeed cfg + 2 * m)) :: [Double])))) y = targetDoubles target df n = VU.length y lr = gbLearningRate cfg @@ -117,21 +154,28 @@ fitGBM cfg target@(Col name) df = , tlMinSamplesSplit = 2 , tlMinLeafSize = 1 , tlMinImpurityDecrease = 0.0 + , tlL2 = gbL2 cfg + , tlMinChildWeight = gbMinChildWeight cfg } - f0 = case gbLoss cfg of - SquaredError -> VU.sum y / fromIntegral (max 1 n) - LogisticDeviance -> + baseScores = fmap (`targetDoubles` df) (gbBaseScore cfg) + f0 = case (baseScores, gbLoss cfg) of + (Just _, _) -> 0 + (Nothing, SquaredError) -> VU.sum y / fromIntegral (max 1 n) + (Nothing, LogisticDeviance) -> let p = clamp01 (VU.sum y / fromIntegral (max 1 n)) in log (p / (1 - p)) - (trees, scores, usage) = boost 0 (VU.replicate n f0) [] [] M.empty + (trees, scores, usage) = boost 0 (fromMaybe (VU.replicate n f0) baseScores) [] [] M.empty boost !m fScores ts ss usageAcc | m >= gbNEstimators cfg = (ts, ss, usageAcc) | otherwise = - let (target', weights) = newtonStep (gbLoss cfg) y fScores - (tree, pred) = fitBinnedTree limits binned target' weights + let (target', weights0) = newtonStep (gbLoss cfg) y fScores + weights = case rowMask m of + Nothing -> weights0 + Just mask -> Just (VU.zipWith (*) mask (fromMaybe (VU.replicate n 1) weights0)) + (tree, pred) = fitBinnedTreeOn (featuresFor m) limits binned target' weights fScores' = VU.zipWith (\f p -> f + lr * p) fScores pred - score = lossValue (gbLoss cfg) y fScores' - usage' = foldr (\c -> M.insertWith (+) c 1) usageAcc (treeColumns tree) + !score = lossValue (gbLoss cfg) y fScores' + !usage' = foldr (\c -> M.insertWith (+) c 1) usageAcc (treeColumns tree) in boost (m + 1) fScores' (tree : ts) (score : ss) usage' fitGBM _ expr _ = throw (NonColumnReferenceException ("fitGBM: " <> T.pack (show expr))) @@ -195,8 +239,9 @@ gbExprAtStage k m stageExpr :: Int -> GBModel -> Expr Double stageExpr k m = - foldr ((.+.) . scaled) (F.lit (gbInit m)) (take k (V.toList (gbTrees m))) + foldr ((.+.) . scaled) start (take k (V.toList (gbTrees m))) where + start = maybe (F.lit (gbInit m)) (\b -> b .+. F.lit (gbInit m)) (gbBase m) scaled t = F.lit (gbRate m) .*. treeToExpr t -- | Probability expression for classification: @sigmoid(score)@. @@ -206,3 +251,11 @@ gbProbaExpr m = F.lit 1 / (F.lit 1 + exp (negate (gbExpr m))) -- | Decision expression for classification: positive class when score > 0. gbDecisionExpr :: GBModel -> Expr Bool gbDecisionExpr m = gbExpr m .>. F.lit 0 + +-- | A uniformly chosen subset of the given size fraction (at least one element), in order. +keepFraction :: Double -> StdGen -> VU.Vector Int -> VU.Vector Int +keepFraction frac g xs = + let k = max 1 (round (frac * fromIntegral (VU.length xs))) + keyed = zip (take (VU.length xs) (randoms g :: [Double])) (VU.toList xs) + chosen = map snd (take k (M.toAscList (M.fromList keyed))) + in VU.fromList (M.keys (M.fromList [(x, ()) | x <- chosen])) diff --git a/dataframe-learn/tests-internal/DecisionTree.hs b/dataframe-learn/tests-internal/DecisionTree.hs index 21e55257..0a8b6729 100644 --- a/dataframe-learn/tests-internal/DecisionTree.hs +++ b/dataframe-learn/tests-internal/DecisionTree.hs @@ -41,6 +41,7 @@ import DataFrame.DecisionTree.Predict ( partitionIndices, predictWithTree, ) +import DataFrame.DecisionTree.Linear (fitLinearCandidate) import DataFrame.DecisionTree.Tao (taoIteration, taoOptimize) import DataFrame.DecisionTree.Types (CarePoint (..), Direction (..)) import DataFrame.Expression.Operators @@ -1357,9 +1358,30 @@ threshSemanticPreservation = TestCase $ do -- Test list ------------------------------------------------------------------------ +-- Regression: on Int features the hyperplane used to be rebuilt over `Col c :: Expr Double`, +-- which cannot evaluate, so every oblique candidate was silently dropped. +obliqueOnIntColumns :: Test +obliqueOnIntColumns = TestCase $ do + let as = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 2, 4, 6, 8, 1, 3, 5, 7, 9] :: [Int] + bs = [9, 7, 5, 3, 1, 8, 6, 4, 2, 0, 1, 3, 5, 7, 9, 0, 2, 4, 6, 8] :: [Int] + labels = [if a + b > 9 then "yes" else "no" | (a, b) <- zip as bs] :: [T.Text] + df = D.fromColumns [("a", DI.fromList as), ("b", DI.fromList bs), ("label", DI.fromList labels)] + cps = [CarePoint i (if l == "yes" then GoRight else GoLeft) | (i, l) <- zip [0 ..] labels] + case fitLinearCandidate defaultTreeConfig "label" df cps of + Nothing -> assertFailure "no oblique candidate on Int features" + Just cond -> case interpret @Bool df cond of + Left err -> assertFailure ("oblique condition does not evaluate: " ++ show err) + Right (DI.TColumn col) -> case DI.toVector @Bool @V.Vector col of + Left err -> assertFailure (show err) + Right v -> do + let sides = V.toList (v :: V.Vector Bool) + assertBool "both sides non-empty" (or sides && not (and sides)) + assertBool "uses both features" (length (getColumns cond) == 2) + tests :: [Test] tests = - [ TestLabel "carePointsBothWrong" carePointsBothWrong + [ TestLabel "obliqueOnIntColumns" obliqueOnIntColumns + , TestLabel "carePointsBothWrong" carePointsBothWrong , TestLabel "carePointsLeftCorrect" carePointsLeftCorrect , TestLabel "carePointsRightCorrect" carePointsRightCorrect , TestLabel "carePointsMixed" carePointsMixed From 375220630d9f98f14f4ec6d7d44e454c5f06d14f Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Wed, 7 Oct 2026 02:54:10 -0700 Subject: [PATCH 2/2] fix: Linting --- .../DataFrame/Internal/DataFrame.hs | 2 +- .../DataFrame/DecisionTree/Linear.hs | 24 +++++++++++---- dataframe-learn/src/DataFrame/Boosting/GBM.hs | 29 +++++++++++++++---- .../tests-internal/DecisionTree.hs | 11 +++++-- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/dataframe-core/src-internal/DataFrame/Internal/DataFrame.hs b/dataframe-core/src-internal/DataFrame/Internal/DataFrame.hs index 0b827f11..83f3d6fd 100644 --- a/dataframe-core/src-internal/DataFrame/Internal/DataFrame.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/DataFrame.hs @@ -45,7 +45,7 @@ import qualified Data.Vector.Unboxed as VU import Control.Exception (throw) import Data.Function (on) -import Data.List (sortBy, (\\), foldl') +import Data.List (foldl', sortBy, (\\)) import Data.Maybe (fromMaybe) import Data.Type.Equality ( TestEquality (testEquality), diff --git a/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs b/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs index 426306cd..bc92b372 100644 --- a/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs +++ b/dataframe-learn/src-internal/DataFrame/DecisionTree/Linear.hs @@ -21,10 +21,10 @@ import DataFrame.DecisionTree.Types ( Direction (..), TreeConfig (..), ) -import DataFrame.Internal.Column (TypedColumn (..), toVector) -import DataFrame.Internal.DataFrame (DataFrame) import DataFrame.Expression.Operators ((.*.), (.+.), (.>.)) import qualified DataFrame.Functions as F +import DataFrame.Internal.Column (TypedColumn (..), toVector) +import DataFrame.Internal.DataFrame (DataFrame) import DataFrame.Internal.Expression (Expr, getColumns) import DataFrame.Internal.Interpreter (interpret) import qualified DataFrame.LinearSolver as LS @@ -60,10 +60,16 @@ fitLinearCandidate cfg target df carePoints = that produced them, so the hyperplane can be rebuilt over Int or nullable columns. -} materializedFeatures :: - T.Text -> DataFrame -> [CarePoint] -> [(Expr Double, (T.Text, VU.Vector Double))] + T.Text -> + DataFrame -> + [CarePoint] -> + [(Expr Double, (T.Text, VU.Vector Double))] materializedFeatures target df carePoints = mapMaybe - (\ne -> (\m -> (doubleExprOf ne (snd m), m)) <$> materializeFeatureForCare df carePoints ne) + ( \ne -> + (\m -> (doubleExprOf ne (snd m), m)) + <$> materializeFeatureForCare df carePoints ne + ) (featureCols target df) -- | The expression behind a feature; a nullable one takes the care-point mean, as its values did. @@ -75,7 +81,10 @@ featureCols :: T.Text -> DataFrame -> [NumExpr] featureCols target df = filter (notElem target . numExprCols) (numericCols df) linearFromFeatures :: - TreeConfig -> [CarePoint] -> [(Expr Double, (T.Text, VU.Vector Double))] -> Maybe (Expr Bool) + TreeConfig -> + [CarePoint] -> + [(Expr Double, (T.Text, VU.Vector Double))] -> + Maybe (Expr Bool) linearFromFeatures cfg carePoints feats | VU.all (== 0) weights = Nothing | degenerateHyperplane rows weights (LS.lmIntercept model) = Nothing @@ -83,7 +92,10 @@ linearFromFeatures cfg carePoints feats where mats = map snd feats hyperplane = - foldl (\acc (w, e) -> acc .+. (F.lit w .*. e)) (F.lit (LS.lmIntercept model)) [(w, e) | (w, (e, _)) <- zip (VU.toList weights) feats, w /= 0] + foldl + (\acc (w, e) -> acc .+. (F.lit w .*. e)) + (F.lit (LS.lmIntercept model)) + [(w, e) | (w, (e, _)) <- zip (VU.toList weights) feats, w /= 0] .>. F.lit (0 :: Double) rows = careRowsFromFeatures (length carePoints) mats labels = careLabels carePoints diff --git a/dataframe-learn/src/DataFrame/Boosting/GBM.hs b/dataframe-learn/src/DataFrame/Boosting/GBM.hs index 907d5a8e..844e0e88 100644 --- a/dataframe-learn/src/DataFrame/Boosting/GBM.hs +++ b/dataframe-learn/src/DataFrame/Boosting/GBM.hs @@ -31,6 +31,7 @@ import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import DataFrame.Errors (DataFrameException (..)) +import Data.Maybe (fromMaybe) import DataFrame.DecisionTree.Cart (cartFeaturesByColumn) import DataFrame.DecisionTree.Fit (treeToExpr) import DataFrame.DecisionTree.Histogram ( @@ -40,13 +41,12 @@ import DataFrame.DecisionTree.Histogram ( ) import DataFrame.DecisionTree.Types (Tree) import DataFrame.Expression.Operators ((.*.), (.+.), (.>.)) -import Data.Maybe (fromMaybe) -import System.Random (StdGen, mkStdGen, randoms) import DataFrame.Featurize.Internal (targetDoubles) import qualified DataFrame.Functions as F import DataFrame.Internal.DataFrame (DataFrame) import DataFrame.Internal.Expression (Expr (..), getColumns) import DataFrame.Model +import System.Random (StdGen, mkStdGen, randoms) -- | The boosting loss. data GBLoss = SquaredError | LogisticDeviance @@ -67,8 +67,9 @@ data GBConfig = GBConfig , gbMinChildWeight :: !Double -- ^ Smallest Hessian sum a leaf may have (XGBoost's @min_child_weight@). , gbFeatureGroups :: ![[T.Text]] - -- ^ Tree @m@ splits only on columns of group @m mod (number of groups)@. - -- Empty: every tree may use every column. + {- ^ Tree @m@ splits only on columns of group @m mod (number of groups)@. + Empty: every tree may use every column. + -} , gbBaseScore :: !(Maybe (Expr Double)) -- ^ Boost from these per-row scores instead of a constant; the prediction adds them back. , gbSubsample :: !Double @@ -141,10 +142,26 @@ fitGBM cfg target@(Col name) df = let allowed = case groupFeatures of [] -> allFeatures gs -> gs !! (m `mod` length gs) - in if gbColsample cfg >= 1 then allowed else keepFraction (gbColsample cfg) (mkStdGen (7919 * gbSeed cfg + 2 * m + 1)) allowed + in if gbColsample cfg >= 1 + then allowed + else + keepFraction + (gbColsample cfg) + (mkStdGen (7919 * gbSeed cfg + 2 * m + 1)) + allowed rowMask m | gbSubsample cfg >= 1 = Nothing - | otherwise = Just (VU.fromList (take n (map (\u -> if u < gbSubsample cfg then 1 else 0) (randoms (mkStdGen (7919 * gbSeed cfg + 2 * m)) :: [Double])))) + | otherwise = + Just + ( VU.fromList + ( take + n + ( map + (\u -> if u < gbSubsample cfg then 1 else 0) + (randoms (mkStdGen (7919 * gbSeed cfg + 2 * m)) :: [Double]) + ) + ) + ) y = targetDoubles target df n = VU.length y lr = gbLearningRate cfg diff --git a/dataframe-learn/tests-internal/DecisionTree.hs b/dataframe-learn/tests-internal/DecisionTree.hs index 0a8b6729..76875020 100644 --- a/dataframe-learn/tests-internal/DecisionTree.hs +++ b/dataframe-learn/tests-internal/DecisionTree.hs @@ -25,6 +25,7 @@ import DataFrame.DecisionTree.Fit ( probExprs, probsFromIndices, ) +import DataFrame.DecisionTree.Linear (fitLinearCandidate) import DataFrame.DecisionTree.Numeric ( NumExpr (NMaybeDouble), generateNumericConds, @@ -41,7 +42,6 @@ import DataFrame.DecisionTree.Predict ( partitionIndices, predictWithTree, ) -import DataFrame.DecisionTree.Linear (fitLinearCandidate) import DataFrame.DecisionTree.Tao (taoIteration, taoOptimize) import DataFrame.DecisionTree.Types (CarePoint (..), Direction (..)) import DataFrame.Expression.Operators @@ -1365,8 +1365,13 @@ obliqueOnIntColumns = TestCase $ do let as = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 2, 4, 6, 8, 1, 3, 5, 7, 9] :: [Int] bs = [9, 7, 5, 3, 1, 8, 6, 4, 2, 0, 1, 3, 5, 7, 9, 0, 2, 4, 6, 8] :: [Int] labels = [if a + b > 9 then "yes" else "no" | (a, b) <- zip as bs] :: [T.Text] - df = D.fromColumns [("a", DI.fromList as), ("b", DI.fromList bs), ("label", DI.fromList labels)] - cps = [CarePoint i (if l == "yes" then GoRight else GoLeft) | (i, l) <- zip [0 ..] labels] + df = + D.fromColumns + [("a", DI.fromList as), ("b", DI.fromList bs), ("label", DI.fromList labels)] + cps = + [ CarePoint i (if l == "yes" then GoRight else GoLeft) + | (i, l) <- zip [0 ..] labels + ] case fitLinearCandidate defaultTreeConfig "label" df cps of Nothing -> assertFailure "no oblique candidate on Int features" Just cond -> case interpret @Bool df cond of