Add Ridge Regression to Machine Learning (#12246)

* Fix issue #12108: Added Ridge Regression to Machine Learning

* Added type hints and minor case improvements

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Resolved ruff checks

* Added doctests

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Resolved ruff checks

* Resolved mypy checks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Resolved ruff checks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* updating DIRECTORY.md

* Comments should not force line wrapping

* updating DIRECTORY.md

* Fix feature scaling variable assignment in predictions

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Christian Clauss <cclauss@me.com>
Co-authored-by: cclauss <cclauss@users.noreply.github.com>

Author: 3 people

DIRECTORY.md

@@ -741,6 +741,7 @@
   * [Q Learning](machine_learning/q_learning.py)
   * [Random Forest Classifier](machine_learning/random_forest_classifier.py)
   * [Random Forest Regressor](machine_learning/random_forest_regressor.py)
+  * [Ridge Regression](machine_learning/ridge_regression.py)
   * [Rmsprop](machine_learning/rmsprop.py)
   * [Scoring Functions](machine_learning/scoring_functions.py)
   * [Self Organizing Map](machine_learning/self_organizing_map.py)

machine_learning/ridge_regression.py

@@ -0,0 +1,172 @@
+import numpy as np
+import pandas as pd
+
+
+class RidgeRegression:
+    def __init__(
+        self, alpha: float = 0.001, lambda_: float = 0.1, iterations: int = 1000
+    ) -> None:
+        """
+        Ridge Regression Constructor
+        :param alpha: Learning rate for gradient descent
+        :param lambda_: Regularization parameter (L2 regularization)
+        :param iterations: Number of iterations for gradient descent
+        """
+        self.alpha = alpha
+        self.lambda_ = lambda_
+        self.iterations = iterations
+        self.theta: np.ndarray | None = None
+
+    def feature_scaling(
+        self, features: np.ndarray
+    ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+        """
+        Normalize features to have mean 0 and standard deviation 1.
+
+        :param features: Input features, shape (m, n)
+        :return: Tuple containing:
+            - Scaled features
+            - Mean of each feature
+            - Standard deviation of each feature
+
+        Example:
+        >>> rr = RidgeRegression()
+        >>> features = np.array([[1, 2], [2, 3], [4, 6]])
+        >>> scaled_features, mean, std = rr.feature_scaling(features)
+        >>> np.allclose(scaled_features.mean(axis=0), 0)
+        True
+        >>> np.allclose(scaled_features.std(axis=0), 1)
+        True
+        """
+        mean = np.mean(features, axis=0)
+        std = np.std(features, axis=0)
+
+        # Avoid division by zero for constant features (std = 0)
+        std[std == 0] = 1  # Set std=1 for constant features to avoid NaN
+
+        scaled_features = (features - mean) / std
+        return scaled_features, mean, std
+
+    def fit(self, features: np.ndarray, target: np.ndarray) -> None:
+        """
+        Fit the Ridge Regression model to the training data.
+
+        :param features: Input features, shape (m, n)
+        :param target: Target values, shape (m,)
+
+        Example:
+        >>> rr = RidgeRegression(alpha=0.01, lambda_=0.1, iterations=10)
+        >>> features = np.array([[1, 2], [2, 3], [4, 6]])
+        >>> target = np.array([1, 2, 3])
+        >>> rr.fit(features, target)
+        >>> rr.theta is not None
+        True
+        """
+        # Normalize features
+        features_scaled, _mean, _std = self.feature_scaling(features)
+        m, n = features_scaled.shape
+        self.theta = np.zeros(n)  # Initialize weights to zeros
+
+        for _ in range(self.iterations):
+            predictions = features_scaled.dot(self.theta)
+            error = predictions - target
+
+            # Compute gradient with L2 regularization
+            gradient = (features_scaled.T.dot(error) + self.lambda_ * self.theta) / m
+            self.theta -= self.alpha * gradient  # Update weights
+
+    def predict(self, features: np.ndarray) -> np.ndarray:
+        """
+        Predict values using the trained model.
+
+        :param features: Input features, shape (m, n)
+        :return: Predicted values, shape (m,)
+
+        Example:
+        >>> rr = RidgeRegression(alpha=0.01, lambda_=0.1, iterations=10)
+        >>> features = np.array([[1, 2], [2, 3], [4, 6]])
+        >>> target = np.array([1, 2, 3])
+        >>> rr.fit(features, target)
+        >>> predictions = rr.predict(features)
+        >>> predictions.shape == target.shape
+        True
+        """
+        if self.theta is None:
+            raise ValueError("Model is not trained yet. Call the `fit` method first.")
+
+        # Scale features using training data
+        features_scaled, _mean, _std = self.feature_scaling(features)
+        return features_scaled.dot(self.theta)
+
+    def compute_cost(self, features: np.ndarray, target: np.ndarray) -> float:
+        """
+        Compute the cost function with regularization.
+
+        :param features: Input features, shape (m, n)
+        :param target: Target values, shape (m,)
+        :return: Computed cost
+
+        Example:
+        >>> rr = RidgeRegression(alpha=0.01, lambda_=0.1, iterations=10)
+        >>> features = np.array([[1, 2], [2, 3], [4, 6]])
+        >>> target = np.array([1, 2, 3])
+        >>> rr.fit(features, target)
+        >>> cost = rr.compute_cost(features, target)
+        >>> isinstance(cost, float)
+        True
+        """
+        if self.theta is None:
+            raise ValueError("Model is not trained yet. Call the `fit` method first.")
+
+        # Scale features using training data
+        features_scaled, _mean, _std = self.feature_scaling(features)
+        m = len(target)
+        predictions = features_scaled.dot(self.theta)
+        cost = (1 / (2 * m)) * np.sum((predictions - target) ** 2) + (
+            self.lambda_ / (2 * m)
+        ) * np.sum(self.theta**2)
+        return cost
+
+    def mean_absolute_error(self, y_true: np.ndarray, y_pred: np.ndarray) -> float:
+        """
+        Compute Mean Absolute Error (MAE) between true and predicted values.
+
+        :param y_true: Actual target values, shape (m,)
+        :param y_pred: Predicted target values, shape (m,)
+        :return: MAE
+
+        Example:
+        >>> rr = RidgeRegression(alpha=0.01, lambda_=0.1, iterations=10)
+        >>> y_true = np.array([1, 2, 3])
+        >>> y_pred = np.array([1.1, 2.1, 2.9])
+        >>> mae = rr.mean_absolute_error(y_true, y_pred)
+        >>> isinstance(mae, float)
+        True
+        """
+        return np.mean(np.abs(y_true - y_pred))
+
+
+# Example usage
+if __name__ == "__main__":
+    # Load dataset
+    data = pd.read_csv(
+        "https://raw.githubusercontent.com/yashLadha/The_Math_of_Intelligence/master/Week1/ADRvsRating.csv"
+    )
+    data_x = data[["Rating"]].to_numpy()  # Feature: Rating
+    data_y = data["ADR"].to_numpy()  # Target: ADR
+    data_y = (data_y - np.mean(data_y)) / np.std(data_y)
+
+    # Add bias term (intercept) to the feature matrix
+    data_x = np.c_[np.ones(data_x.shape[0]), data_x]  # Add intercept term
+
+    # Initialize and train the Ridge Regression model
+    model = RidgeRegression(alpha=0.01, lambda_=0.1, iterations=1000)
+    model.fit(data_x, data_y)
+
+    # Predictions
+    predictions = model.predict(data_x)
+
+    # Results
+    print("Optimized Weights:", model.theta)
+    print("Cost:", model.compute_cost(data_x, data_y))
+    print("Mean Absolute Error:", model.mean_absolute_error(data_y, predictions))