From 1956c787acd18df5d33f3383c22f1250920ddf95 Mon Sep 17 00:00:00 2001 From: Divakant Pandey Date: Tue, 3 Nov 2026 18:01:48 +0530 Subject: [PATCH] feat(linfa-linear): support sample weights in LinearRegression Support Weighted Least Squares (WLS) when sample weights are attached to DatasetBase. Scale design matrix rows and targets by the square root of sample weights prior to solving least squares, validate non-negative weights, and add unit tests covering outlier filtering and uniform weight equivalence. --- algorithms/linfa-linear/src/error.rs | 2 + algorithms/linfa-linear/src/ols.rs | 190 ++++++++++++++++++++++++++- 2 files changed, 185 insertions(+), 7 deletions(-) diff --git a/algorithms/linfa-linear/src/error.rs b/algorithms/linfa-linear/src/error.rs index ae907fd8d..10f6c4554 100644 --- a/algorithms/linfa-linear/src/error.rs +++ b/algorithms/linfa-linear/src/error.rs @@ -19,6 +19,8 @@ pub enum LinearError { NotEnoughTargets, #[error("penalty should be positive, but is {0}")] InvalidPenalty(F), + #[error("sample weights must be non-negative, but found {0}")] + InvalidWeights(F), #[error("tweedie distribution power should not be in (0, 1), but is {0}")] InvalidTweediePower(F), #[error("some value(s) of y are out of the valid range for power value {0}")] diff --git a/algorithms/linfa-linear/src/ols.rs b/algorithms/linfa-linear/src/ols.rs index 3f915c0cc..42013de46 100644 --- a/algorithms/linfa-linear/src/ols.rs +++ b/algorithms/linfa-linear/src/ols.rs @@ -21,7 +21,7 @@ use linfa::traits::{Fit, PredictInplace}; derive(Serialize, Deserialize), serde(crate = "serde_crate") )] -/// An ordinary least squares univariate linear regression model. +/// An ordinary and weighted least squares univariate linear regression model. /// /// Given predictors `x` and responses `y` ordinary least squares linear /// regression estimates a model of the form @@ -30,7 +30,8 @@ use linfa::traits::{Fit, PredictInplace}; /// /// by finding a matrix `W` and a vector `b` which minimize the sum of the /// squared L_2 norms `||y_j - x_jW - b||_2^2` for a dataset -/// `{(x_j, y_j) for j in 1..=n_samples}`. +/// `{(x_j, y_j) for j in 1..=n_samples}`. When sample weights `w` are provided, +/// it minimizes the weighted sum of squared residuals `w_j ||y_j - x_jW - b||_2^2`. /// /// The algorithm is only implemented for _univariate_ regression. This means /// that `b` and `y` are scalars and `W` is just one column. @@ -94,7 +95,8 @@ impl, T: AsSingleTargets> type Object = FittedLinearRegression; /// Fit a linear regression model given a feature matrix `X` and a target - /// variable `y`. + /// variable `y`. If sample weights are attached to `dataset` via + /// `.with_weights()`, Weighted Least Squares (WLS) is performed. /// /// The feature matrix `X` must have shape `(n_samples, n_features)` /// @@ -114,14 +116,13 @@ impl, T: AsSingleTargets> if self.fit_intercept { let X = concatenate(Axis(1), &[X.view(), Array2::ones((X.nrows(), 1)).view()]).unwrap(); - let params: Array1 = solve_least_squares(X, y.to_owned())?; + let (X, y) = apply_weights(X, y.to_owned(), dataset.weights())?; + let params: Array1 = solve_least_squares(X, y)?; let intercept = *params.last().unwrap(); let params = params.slice(s![..params.len() - 1]).to_owned(); Ok(FittedLinearRegression { intercept, params }) } else { - // `LeastSquaresSvdInto` needs a mutable reference to the data and `dataset` is taken - // by reference. Therefore copy the problem matrix and target vector. - let (X, y) = (X.to_owned(), y.to_owned()); + let (X, y) = apply_weights(X.to_owned(), y.to_owned(), dataset.weights())?; Ok(FittedLinearRegression { intercept: F::cast(0), @@ -131,6 +132,29 @@ impl, T: AsSingleTargets> } } +fn apply_weights( + mut X: Array2, + mut y: Array1, + weights: Option<&[f32]>, +) -> Result<(Array2, Array1), F> { + if let Some(weights) = weights { + let n_samples = X.nrows(); + if weights.len() != n_samples { + return Err(linfa::Error::MismatchedShapes(weights.len(), n_samples).into()); + } + for (i, (&w, mut row)) in weights.iter().zip(X.rows_mut()).enumerate() { + let w_f = F::cast(w); + if w_f < F::zero() { + return Err(LinearError::InvalidWeights(w_f)); + } + let sqrt_w = w_f.sqrt(); + row.mapv_inplace(|v| v * sqrt_w); + y[i] *= sqrt_w; + } + } + Ok((X, y)) +} + /// Find the b that minimizes the 2-norm of X b - y /// by using the least_squares solver from ndarray-linalg fn solve_least_squares(mut X: Array, mut y: Array) -> Result, F> @@ -195,6 +219,158 @@ mod tests { use linfa::{traits::Predict, Dataset}; use ndarray::array; + #[test] + fn fits_weighted_least_squares_with_outlier() { + let dataset = Dataset::new(array![[0f64], [1.], [2.]], array![1., 3., 1000.]) + .with_weights(array![1.0f32, 1.0f32, 0.0f32]); + + let lin_reg = LinearRegression::new(); + let model = lin_reg.fit(&dataset).unwrap(); + + assert_abs_diff_eq!(model.params(), &array![2.0], epsilon = 1e-10); + assert_abs_diff_eq!(model.intercept(), &1.0, epsilon = 1e-10); + } + + #[test] + fn fits_weighted_least_squares_without_intercept() { + // Without intercept, passing through origin + // Point (1, 1) and Point (2, 8) + // If unweighted: min (b - 1)^2 + (2b - 8)^2 => 2(b - 1) + 4(2b - 8) = 0 => 10b - 34 = 0 => b = 3.4 + // If weights are [1.0, 0.0]: b = 1.0 + // If weights are [0.0, 1.0]: b = 4.0 + let data = array![[1f64], [2.]]; + let targets = array![1., 8.]; + + let model_unweighted = LinearRegression::new() + .with_intercept(false) + .fit(&Dataset::new(data.clone(), targets.clone())) + .unwrap(); + assert_abs_diff_eq!(model_unweighted.params(), &array![3.4], epsilon = 1e-10); + + let model_w0 = LinearRegression::new() + .with_intercept(false) + .fit(&Dataset::new(data.clone(), targets.clone()).with_weights(array![1.0, 0.0])) + .unwrap(); + assert_abs_diff_eq!(model_w0.params(), &array![1.0], epsilon = 1e-10); + + let model_w1 = LinearRegression::new() + .with_intercept(false) + .fit(&Dataset::new(data, targets).with_weights(array![0.0, 1.0])) + .unwrap(); + assert_abs_diff_eq!(model_w1.params(), &array![4.0], epsilon = 1e-10); + } + + #[test] + fn fits_weighted_least_squares_uniform_weights_matches_unweighted() { + let dataset_unweighted = Dataset::new(array![[0f64], [1.], [2.]], array![1., 2., 4.]); + let dataset_weighted = dataset_unweighted + .clone() + .with_weights(array![5.0f32, 5.0f32, 5.0f32]); + + let model1 = LinearRegression::new().fit(&dataset_unweighted).unwrap(); + let model2 = LinearRegression::new().fit(&dataset_weighted).unwrap(); + + assert_abs_diff_eq!(model1.params(), model2.params(), epsilon = 1e-10); + assert_abs_diff_eq!(model1.intercept(), model2.intercept(), epsilon = 1e-10); + } + + #[test] + fn fits_weighted_least_squares_rejects_negative_weights() { + let dataset = Dataset::new(array![[0f64], [1.]], array![1., 2.]) + .with_weights(array![1.0f32, -0.5f32]); + + let res = LinearRegression::new().fit(&dataset); + assert!(matches!(res, Err(LinearError::InvalidWeights(w)) if w < 0.0)); + } + + #[test] + fn fits_weighted_least_squares_rejects_mismatched_weights_len() { + let dataset = + Dataset::new(array![[0f64], [1.]], array![1., 2.]).with_weights(array![1.0f32]); + + let res = LinearRegression::new().fit(&dataset); + assert!(matches!( + res, + Err(LinearError::BaseCrate(linfa::Error::MismatchedShapes(1, 2))) + )); + } + + #[test] + fn fits_weighted_least_squares_multivariate() { + let x = array![ + [1.0f64, 2.0], + [2.0, 0.0], + [0.0, 1.0], + [3.0, 4.0], + [10.0, 20.0], + [-5.0, 15.0] + ]; + let y = array![1.0, 9.0, 2.0, -1.0, 9999.0, -8888.0]; + let weights = array![1.0f32, 1.0, 1.0, 1.0, 0.0, 0.0]; + + let dataset = Dataset::new(x, y).with_weights(weights); + let model = LinearRegression::new().fit(&dataset).unwrap(); + + assert_abs_diff_eq!(model.params(), &array![2.0, -3.0], epsilon = 1e-10); + assert_abs_diff_eq!(model.intercept(), &5.0, epsilon = 1e-10); + } + + #[test] + fn fits_weighted_least_squares_f32() { + let dataset = Dataset::new(array![[0.0f32], [1.0], [2.0]], array![1.0f32, 3.0, 500.0]) + .with_weights(array![1.0f32, 1.0, 0.0]); + + let model = LinearRegression::new().fit(&dataset).unwrap(); + + assert_abs_diff_eq!(model.params(), &array![2.0f32], epsilon = 1e-5); + assert_abs_diff_eq!(model.intercept(), &1.0f32, epsilon = 1e-5); + } + + #[test] + fn fits_weighted_least_squares_normal_equations_optimality() { + let x = array![ + [1.0f64, 2.0], + [2.0, 1.5], + [3.0, 0.5], + [4.0, 3.2], + [5.0, 2.1] + ]; + let y = array![2.5f64, 3.1, 4.8, 6.2, 7.9]; + let weights = array![0.5f32, 1.8, 3.2, 0.7, 2.4]; + + let dataset = Dataset::new(x.clone(), y.clone()).with_weights(weights.clone()); + let model = LinearRegression::new().fit(&dataset).unwrap(); + + let preds = model.predict(&x); + let residuals = &y - &preds; + + let mut intercept_grad = 0.0; + for (&w, &r) in weights.iter().zip(residuals.iter()) { + intercept_grad += (w as f64) * r; + } + assert_abs_diff_eq!(intercept_grad, 0.0, epsilon = 1e-10); + + for j in 0..x.ncols() { + let mut feat_grad = 0.0; + for (i, (&w, &r)) in weights.iter().zip(residuals.iter()).enumerate() { + feat_grad += (w as f64) * r * x[[i, j]]; + } + assert_abs_diff_eq!(feat_grad, 0.0, epsilon = 1e-10); + } + } + + #[test] + fn fits_weighted_least_squares_dataset_view() { + let dataset = Dataset::new(array![[0.0f64], [1.0], [2.0]], array![1.0, 3.0, 1000.0]) + .with_weights(array![1.0f32, 1.0, 0.0]); + + let dataset_view = dataset.view(); + let model = LinearRegression::new().fit(&dataset_view).unwrap(); + + assert_abs_diff_eq!(model.params(), &array![2.0], epsilon = 1e-10); + assert_abs_diff_eq!(model.intercept(), &1.0, epsilon = 1e-10); + } + #[test] fn autotraits() { fn has_autotraits() {}