MSE

The regularized MSE regression solves the following optimization problem:

\[\min_{\mathbf{\beta} \in \mathbb{R}^d} C \sum_{i=1}^n (y_i - \mathbf{x}_i^\top \mathbf{\beta})^2 + \frac{1}{2}\|\mathbf{\beta}\|_2^2,\]

where \(\mathbf{x}_i \in \mathbb{R}^d\) is a feature vector, and \(y_i \in \mathbb{R}\) is the response variable.

Note. Since the squared loss is a plq function, we can optimize it using rehline.plq_Ridge_Regressor. Moreover, this wrapper adapts the plqERM_Ridge into a regressor, compatible with the scikit-learn API.

## install rehline
%pip install rehline -q

import warnings

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.datasets import make_regression
from sklearn.preprocessing import StandardScaler

# Simulate data
np.random.seed(42)
scaler_mse = StandardScaler()

n, d = 10000, 5
X, y = make_regression(n_samples=n, n_features=d, noise=5.0)
X = scaler_mse.fit_transform(X)
y = y / y.std()

## solve MSE Regression via `plq_Ridge_Regressor`
from rehline import plq_Ridge_Regressor

clf = plq_Ridge_Regressor(loss={"name": "mse"}, C=1.0)
clf.fit(X=X, y=y)

plq_Ridge_Regressor(loss={'name': 'mse'})

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

## plot MSE results
warnings.filterwarnings("ignore", "is_categorical_dtype")

n_sample = 200
X_sample, y_sample = X[:n_sample], y[:n_sample]
mse_sample = clf.predict(X_sample)

df = pd.DataFrame({"x0": X_sample[:, 0], "real_y": y_sample, "mse": mse_sample})
df = df.melt(id_vars="x0")

sns.scatterplot(data=df, x="x0", y="value", hue="variable")
plt.show()

With Pipeline

plq_Ridge_Regressor can be integrated into a scikit-learn Pipeline to streamline preprocessing including scaling.

import warnings

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.datasets import make_regression

# Simulate data
np.random.seed(42)

n, d = 10000, 5
X, y = make_regression(n_samples=n, n_features=d, noise=5.0)
y = y / y.std()

## solve MSE Regression via `plq_Ridge_Regressor`
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

from rehline import plq_Ridge_Regressor

pipe = Pipeline([("scaler", StandardScaler()), ("reg", plq_Ridge_Regressor(loss={"name": "mse"}, C=1.0))])
pipe.fit(X, y)

Pipeline(steps=[('scaler', StandardScaler()),
                ('reg', plq_Ridge_Regressor(loss={'name': 'mse'}))])

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

## plot MSE results
warnings.filterwarnings("ignore", "is_categorical_dtype")

n_sample = 200
X_sample, y_sample = X[:n_sample], y[:n_sample]
mse_sample = pipe.predict(X_sample)

df = pd.DataFrame({"x0": X_sample[:, 0], "real_y": y_sample, "mse": mse_sample})
df = df.melt(id_vars="x0")

sns.scatterplot(data=df, x="x0", y="value", hue="variable")
plt.show()

Hyperparameter Tuning with GridSearchCV

Due to its compatibility with the scikit-learn API, GridSearchCV can be applied to determine the optimal hyperparameters for the ReHLine model.

import warnings

from sklearn.metrics import make_scorer, mean_squared_error
from sklearn.model_selection import GridSearchCV

warnings.filterwarnings("ignore")

# Define the parameter grid to search
param_grid = {"reg__C": [0.1, 1.0, 10.0]}

# Use mse to evaluate the performances
mse_scorer = make_scorer(mean_squared_error, greater_is_better=False)

# Create the GridSearchCV objects
grid_mse = GridSearchCV(pipe, param_grid, cv=5, scoring=mse_scorer)

grid_mse.fit(X, y)

# Print the best parameters and scores
print(f"Best params:{grid_mse.best_params_}")
print(f"Best CV Score: {-grid_mse.best_score_:.4f}")

Best params:{'reg__C': 1.0}
Best CV Score: 0.0014

## plot MSE results
n_sample = 200
X_sample, y_sample = X[:n_sample], y[:n_sample]
mae_sample = grid_mse.predict(X_sample)

df = pd.DataFrame({"x0": X_sample[:, 0], "real_y": y_sample, "mse": mse_sample})
df = df.melt(id_vars="x0")

sns.scatterplot(data=df, x="x0", y="value", hue="variable").set_title("MSE(C=1.0)")
plt.show()