"""Chapter 6 lab: Linear Model Selection and Regularization (Python version).

Course: Applied Statistical Learning in R and Python, which follows James, Witten, Hastie,
Tibshirani and Taylor (2023), An Introduction to Statistical Learning with Applications in Python,
Section 6.5, and the R edition of 2021.

Predict a baseball player's Salary from the Hitters data with
  [1] the data;  [2] best subset selection (our own exhaustive search) and Cp, BIC, adjusted R^2;
  [3] forward and backward stepwise selection;  [4] choosing the size by a validation set and by
  10-fold cross-validation;  [5] ridge regression at fixed lambda;  [6] ridge and the lasso on a
  training/test split with lambda chosen by cross-validation;  [7] principal components regression
  and [8] partial least squares.
Only pandas, numpy, statsmodels and scikit-learn are used. Penalties are put on glmnet's scale
(the scale of the R lab, ch06_lab.R), so the two labs print the same coefficients wherever the
computation is deterministic; the random splits use numpy's and scikit-learn's generators and
therefore differ from R's.

Run: python3 ch06_lab.py     (prints the results; writes ../img/ch06/lab-py-*.{png,svg,pdf})
The code is my own.
"""
import os

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import itertools
import warnings

import numpy as np
import pandas as pd
import statsmodels.api as sm
import sklearn.linear_model as skl
import sklearn.model_selection as skm
from sklearn.cross_decomposition import PLSRegression
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

from common import isl_data, savefig, plt, RED, BLUE, GREY

warnings.filterwarnings("ignore", category=UserWarning)
pd.set_option("display.width", 100)
np.set_printoptions(linewidth=100, suppress=True)


def step(k, title):
    print("\n== [%s] %s" % (k, title))


# -- [1] The Hitters data: drop the players whose salary is missing
step(1, "The data")
Hitters = isl_data("Hitters")
print(Hitters.shape, Hitters["Salary"].isna().sum())
Hitters = Hitters.dropna()
print(Hitters.shape)
# The design matrix in the column order of R's model.matrix(): 0/1 dummies for the factors
num = ["AtBat", "Hits", "HmRun", "Runs", "RBI", "Walks", "Years", "CAtBat", "CHits", "CHmRun",
       "CRuns", "CRBI", "CWalks"]
X = Hitters[num].astype(float)
X["LeagueN"] = (Hitters["League"] == "N").astype(float)
X["DivisionW"] = (Hitters["Division"] == "W").astype(float)
for c in ["PutOuts", "Assists", "Errors"]:
    X[c] = Hitters[c].astype(float)
X["NewLeagueN"] = (Hitters["NewLeague"] == "N").astype(float)
y = Hitters["Salary"].to_numpy()
names = list(X.columns)
Xa = X.to_numpy()
n, p = Xa.shape


# -- [2] Best subset selection: the best model of every size, and Cp, BIC, adjusted R^2
step(2, "Best subset selection")
def best_subset(X, y, max_size=None):
    """For each size k, the subset of columns with the smallest RSS (with an intercept).
    Exhaustive: every subset's RSS is y'y - b'G^{-1}b on the centred, scaled data, solved in batches."""
    Z = (X - X.mean(0)) / X.std(0)
    yc = y - y.mean()
    G, b, tss = Z.T @ Z, Z.T @ yc, yc @ yc
    best = []
    for k in range(1, (max_size or X.shape[1]) + 1):
        combos = np.array(list(itertools.combinations(range(X.shape[1]), k)))
        rss = np.empty(len(combos))
        for a in range(0, len(combos), 20000):              # 92,378 subsets of size 9 or 10
            c = combos[a:a + 20000]
            sol = np.linalg.solve(G[c[:, :, None], c[:, None, :]], b[c][..., None])[..., 0]
            rss[a:a + 20000] = tss - (b[c] * sol).sum(1)
        i = np.argmin(rss)
        best.append((list(combos[i]), rss[i]))
    return best, tss


best, tss = best_subset(Xa, y)                               # 2^19 - 1 = 524,287 models
rss = np.array([r for _, r in best])
d = np.arange(1, p + 1)                                      # number of predictors
sigma2 = rss[-1] / (n - p - 1)                               # error variance from the full model
crit = pd.DataFrame({
    "rsq": 1 - rss / tss,
    "adjr2": 1 - (rss / (n - d - 1)) / (tss / (n - 1)),                  # (6.4)
    "Cp": (rss + 2 * d * sigma2) / n,                                      # (6.2)
    "BIC": (rss + np.log(n) * d * sigma2) / n,                             # (6.3)
    "BIC_leaps": n * np.log(rss / tss) + (d + 1) * np.log(n),             # R's summary()$bic
}, index=d)
print(crit["rsq"].round(3).to_numpy())
print({"adjr2": crit["adjr2"].idxmax(), "Cp": crit["Cp"].idxmin(), "BIC": crit["BIC"].idxmin(),
       "BIC_leaps": crit["BIC_leaps"].idxmin()})
cols6 = [names[j] for j in best[5][0]]
fit6 = sm.OLS(y, sm.add_constant(X[cols6])).fit()
print(fit6.params.round(6))

fig, axes = plt.subplots(2, 2, figsize=(8, 6))
for ax, col, lab, best_k in zip(axes.flat, ["rsq", "adjr2", "Cp", "BIC"],
                                ["$R^2$", "Adjusted $R^2$", "$C_p$", "BIC"],
                                [None, crit["adjr2"].idxmax(), crit["Cp"].idxmin(), crit["BIC"].idxmin()]):
    ax.plot(d, crit[col], color="k")
    if best_k is not None:
        ax.plot(best_k, crit[col][best_k], "o", color=RED, ms=8)
    ax.set_xlabel("Number of variables")
    ax.set_ylabel(lab)
fig.tight_layout()
savefig(fig, 6, "lab-py-criteria")


# -- [3] Forward and backward stepwise selection
step(3, "Forward and backward stepwise selection")
def rss_of(X, y, cols):
    A = np.column_stack([np.ones(len(y))] + [X[:, j] for j in cols])
    r = y - A @ np.linalg.lstsq(A, y, rcond=None)[0]
    return r @ r


def forward_stepwise(X, y):
    """Algorithm 6.2: add, one at a time, the variable that lowers the RSS most."""
    path, left = [[]], list(range(X.shape[1]))
    while left:
        j = min(left, key=lambda j: rss_of(X, y, path[-1] + [j]))
        path.append(path[-1] + [j])
        left.remove(j)
    return path                                              # path[k] = the k-variable model


def backward_stepwise(X, y):
    """Algorithm 6.3: drop, one at a time, the variable whose removal raises the RSS least."""
    path = [list(range(X.shape[1]))]
    while path[-1]:
        cur = path[-1]
        j = min(cur, key=lambda j: rss_of(X, y, [c for c in cur if c != j]))
        path.append([c for c in cur if c != j])
    return path[::-1]


fwd, bwd = forward_stepwise(Xa, y), backward_stepwise(Xa, y)
for label, cols in [("best", best[6][0]), ("forward", fwd[7]), ("backward", bwd[7])]:
    c = [names[j] for j in sorted(cols)]
    print(label, sm.OLS(y, sm.add_constant(X[c])).fit().params.round(3).to_dict())
# Cp along the forward path; the minimum is the model the Python edition's forward search stops at
cp_fwd = [(rss_of(Xa, y, fwd[k]) + 2 * k * sigma2) / n for k in range(p + 1)]
k_cp = int(np.argmin(cp_fwd))
print(k_cp, sorted(names[j] for j in fwd[k_cp]))


# -- [4] Choosing the model size by a validation set and by 10-fold cross-validation
step(4, "Validation set and cross-validation")
def predict_subset(Xtr, ytr, Xte, cols):
    A = np.column_stack([np.ones(len(ytr))] + [Xtr[:, j] for j in cols])
    beta = np.linalg.lstsq(A, ytr, rcond=None)[0]
    return np.column_stack([np.ones(len(Xte))] + [Xte[:, j] for j in cols]) @ beta


rng = np.random.default_rng(1)
train = rng.choice([True, False], n)                         # about half the players
best_tr, _ = best_subset(Xa[train], y[train])                # selection on the training part only
val_errors = np.array([np.mean((y[~train] - predict_subset(Xa[train], y[train], Xa[~train], c)) ** 2)
                       for c, _ in best_tr])
print(val_errors.round(0))
print(int(np.argmin(val_errors)) + 1)

K = 10
folds = rng.permutation(np.arange(n) % K)
cv_errors = np.zeros((K, p))
for j in range(K):
    tr, te = folds != j, folds == j
    best_j, _ = best_subset(Xa[tr], y[tr])                   # best subset again inside every fold
    for i, (c, _) in enumerate(best_j):
        cv_errors[j, i] = np.mean((y[te] - predict_subset(Xa[tr], y[tr], Xa[te], c)) ** 2)
mean_cv = cv_errors.mean(0)
print(mean_cv.round(0))
k_cv = int(np.argmin(mean_cv)) + 1
print(k_cv, [names[j] for j in sorted(best[k_cv - 1][0])])

fig, ax = plt.subplots(figsize=(6, 4))
ax.plot(d, val_errors, "o--", color=BLUE, ms=4, label="validation set")
ax.plot(d, mean_cv, "o-", color=RED, ms=4, label="10-fold CV")
ax.set_xlabel("Number of variables")
ax.set_ylabel("Estimated test MSE")
ax.legend()
savefig(fig, 6, "lab-py-cv-size")


# -- [5] Ridge regression at fixed lambda, on glmnet's scale
step(5, "Ridge regression at fixed lambda")
def ridge_glmnet(X, y, lam):
    """Ridge with glmnet's conventions: columns standardised with the 1/n standard deviation, and
    glmnet's lambda equal to (2 sd(y)/n) times the lambda of (6.5) for the standardised columns.
    Returns (intercept, coefficients on the original scale)."""
    scaler = StandardScaler().fit(X)
    alpha = len(y) * lam / y.std()                           # sklearn's Ridge minimises RSS + alpha |b|^2
    fit = skl.Ridge(alpha=alpha).fit(scaler.transform(X), y)
    beta = fit.coef_ / scaler.scale_
    return y.mean() - scaler.mean_ @ beta, beta


grid = 10 ** np.linspace(10, -2, 100)                        # the R lab's grid, largest first
for lam in [grid[49], grid[59]]:
    b0, beta = ridge_glmnet(Xa, y, lam)
    print(round(lam, 2), round(b0, 3), round(np.sqrt(beta @ beta), 2))
b0, beta = ridge_glmnet(Xa, y, grid[49])
print(pd.Series(np.r_[b0, beta], ["(Intercept)"] + names).round(3).to_dict())
b0, beta = ridge_glmnet(Xa, y, 50)
print(pd.Series(np.r_[b0, beta], ["(Intercept)"] + names).round(3).to_dict())


# -- [6] Ridge and the lasso on a training/test split, lambda chosen by 10-fold CV
step(6, "Ridge and the lasso: test error")
split = skm.ShuffleSplit(n_splits=1, test_size=0.5, random_state=0)   # the Python edition's split
tr, te = next(split.split(Xa))
def test_mse(pred):
    return np.mean((pred - y[te]) ** 2)


b0, beta = ridge_glmnet(Xa[tr], y[tr], 4)
print(round(test_mse(b0 + Xa[te] @ beta)))                   # lambda = 4
print(round(test_mse(np.full(len(te), y[tr].mean()))))       # intercept only
ls_fit = skl.LinearRegression().fit(Xa[tr], y[tr])
print(round(test_mse(ls_fit.predict(Xa[te]))))               # least squares
kfold = skm.KFold(10, shuffle=True, random_state=0)
lam_grid = 10 ** np.linspace(5, -2, 100)


def cv_lambda(fit_predict, X, y, lams, cv):
    """Mean CV error for each lambda; standardisation is redone inside every fold, as glmnet does."""
    err = np.zeros((cv.get_n_splits(), len(lams)))
    for f, (a, b) in enumerate(cv.split(X)):
        for i, lam in enumerate(lams):
            err[f, i] = np.mean((y[b] - fit_predict(X[a], y[a], X[b], lam)) ** 2)
    return err.mean(0), err.std(0) / np.sqrt(err.shape[0])


def ridge_fp(Xtr, ytr, Xte, lam):
    b0, beta = ridge_glmnet(Xtr, ytr, lam)
    return b0 + Xte @ beta


def lasso_glmnet(X, y, lam):
    """The lasso on glmnet's scale: sklearn's Lasso minimises RSS/(2n) + alpha |b|_1, which is
    glmnet's criterion when alpha = lambda and the columns are standardised (1/n sd)."""
    scaler = StandardScaler().fit(X)
    fit = skl.Lasso(alpha=lam, tol=1e-10, max_iter=100000).fit(scaler.transform(X), y)
    beta = fit.coef_ / scaler.scale_
    return y.mean() - scaler.mean_ @ beta, beta


def lasso_fp(Xtr, ytr, Xte, lam):
    b0, beta = lasso_glmnet(Xtr, ytr, lam)
    return b0 + Xte @ beta


cv_r, se_r = cv_lambda(ridge_fp, Xa[tr], y[tr], lam_grid, kfold)
lam_r = lam_grid[np.argmin(cv_r)]
b0, beta = ridge_glmnet(Xa[tr], y[tr], lam_r)
print(round(lam_r, 2), round(test_mse(b0 + Xa[te] @ beta)))
b0, beta = ridge_glmnet(Xa, y, lam_r)                        # refit on all the data
print(pd.Series(np.r_[b0, beta], ["(Intercept)"] + names).round(2).to_dict())

lam_grid_l = 10 ** np.linspace(2.5, -2, 80)
cv_l, se_l = cv_lambda(lasso_fp, Xa[tr], y[tr], lam_grid_l, kfold)
lam_l = lam_grid_l[np.argmin(cv_l)]
b0, beta = lasso_glmnet(Xa[tr], y[tr], lam_l)
print(round(lam_l, 2), round(test_mse(b0 + Xa[te] @ beta)))
b0, beta = lasso_glmnet(Xa, y, lam_l)
coef_l = pd.Series(np.r_[b0, beta], ["(Intercept)"] + names)
print(coef_l[coef_l != 0].round(2).to_dict())
print(int((beta != 0).sum()))

fig, axes = plt.subplots(1, 2, figsize=(9, 3.8))
for ax, lams, cv, se, best_lam, title in [(axes[0], lam_grid, cv_r, se_r, lam_r, "Ridge"),
                                          (axes[1], lam_grid_l, cv_l, se_l, lam_l, "Lasso")]:
    ax.errorbar(np.log(lams), cv, yerr=se, color=RED, ecolor="#cccccc", lw=1.4)
    ax.axvline(np.log(best_lam), color="k", ls="--", lw=1)
    ax.set_xlabel(r"$\log(\lambda)$ (glmnet scale)")
    ax.set_ylabel("10-fold CV MSE")
    ax.set_title(title)
fig.tight_layout()
savefig(fig, 6, "lab-py-cv-lambda")

# The same questions as the Python edition asks them: 5-fold CV on all 263 players, with
# scikit-learn's ElasticNetCV (its alpha is glmnet's lambda divided by sd(y) for ridge)
kfold5 = skm.KFold(5, shuffle=True, random_state=0)
alphas = 10 ** np.linspace(8, -2, 100) / y.std()
ridgeCV = Pipeline([("scaler", StandardScaler()),
                    ("ridge", skl.ElasticNetCV(alphas=alphas, l1_ratio=0, cv=kfold5))]).fit(Xa, y)
tuned = ridgeCV.named_steps["ridge"]
print(round(tuned.alpha_, 5), round(tuned.mse_path_.mean(1).min(), 2))
lassoCV = Pipeline([("scaler", StandardScaler()),
                    ("lasso", skl.ElasticNetCV(n_alphas=100, l1_ratio=1, cv=kfold5))]).fit(Xa, y)
tuned = lassoCV.named_steps["lasso"]
print(round(tuned.alpha_, 3), round(tuned.mse_path_.mean(1).min(), 2), int((tuned.coef_ == 0).sum()))

lasso_path = np.array([lasso_glmnet(Xa[tr], y[tr], lam)[1] * Xa[tr].std(0) for lam in lam_grid_l])
fig, ax = plt.subplots(figsize=(6, 4))
ax.plot(np.log(lam_grid_l), lasso_path, lw=1.2)
ax.axhline(0, color=GREY, lw=0.6)
ax.set_xlabel(r"$\log(\lambda)$ (glmnet scale)")
ax.set_ylabel("Standardised coefficients")
savefig(fig, 6, "lab-py-lasso-path")


# -- [7] Principal components regression
step(7, "Principal components regression")
def pcr(M):
    return Pipeline([("scaler", StandardScaler()), ("pca", PCA(n_components=M)),
                     ("ols", skl.LinearRegression())])


pca = PCA().fit(StandardScaler().fit_transform(Xa))
print((100 * pca.explained_variance_ratio_[:5]).round(2))
print((100 * np.cumsum(pca.explained_variance_ratio_)[:5]).round(2))
print(np.array([100 * pcr(M).fit(Xa, y).score(Xa, y) for M in range(1, 6)]).round(2))   # % of Salary


def cv_components(make, X, y, cv, Ms):
    """CV MSE for M = 0 (intercept only) and each M in Ms."""
    err = np.zeros((cv.get_n_splits(), len(Ms) + 1))
    for f, (a, b) in enumerate(cv.split(X)):
        err[f, 0] = np.mean((y[b] - y[a].mean()) ** 2)
        for i, M in enumerate(Ms, 1):
            err[f, i] = np.mean((y[b] - make(M).fit(X[a], y[a]).predict(X[b]).ravel()) ** 2)
    return err.mean(0)


Ms = range(1, p + 1)
cv_pcr_all = cv_components(pcr, Xa, y, kfold, Ms)
print(np.sqrt(cv_pcr_all[:5]).round(1), int(np.argmin(cv_pcr_all)))     # RMSE, like R's summary
cv_pcr = cv_components(pcr, Xa[tr], y[tr], kfold, Ms)
M_pcr = int(np.argmin(cv_pcr))
print(M_pcr, round(test_mse(pcr(M_pcr).fit(Xa[tr], y[tr]).predict(Xa[te]))),
      round(test_mse(pcr(5).fit(Xa[tr], y[tr]).predict(Xa[te]))))


# -- [8] Partial least squares
step(8, "Partial least squares")
def pls(M):
    return PLSRegression(n_components=M, scale=True)


cv_pls = cv_components(pls, Xa[tr], y[tr], kfold, Ms)
M_pls = int(np.argmin(cv_pls))
print(np.sqrt(cv_pls[:5]).round(1), M_pls)
print(round(test_mse(pls(M_pls).fit(Xa[tr], y[tr]).predict(Xa[te]).ravel())),
      round(test_mse(pls(1).fit(Xa[tr], y[tr]).predict(Xa[te]).ravel())))
pls1 = pls(1).fit(Xa, y)
Z = (Xa - Xa.mean(0)) / Xa.std(0, ddof=1)                  # PLSRegression scales with ddof = 1
T, P = pls1.x_scores_, pls1.x_loadings_
x_expl = 100 * np.sum((T @ P.T) ** 2) / np.sum(Z ** 2)       # % of the variance of X captured
print(round(x_expl, 2), round(100 * pls1.score(Xa, y), 2))

fig, axes = plt.subplots(1, 2, figsize=(9, 3.8), sharey=True)
for ax, cv, Mbest, title in [(axes[0], cv_pcr, M_pcr, "PCR"), (axes[1], cv_pls, M_pls, "PLS")]:
    ax.plot(range(0, p + 1), cv, "o-", color=RED, ms=4)
    ax.axvline(Mbest, color="k", ls="--", lw=1)
    ax.set_xlabel("Number of components M")
    ax.set_title(title)
axes[0].set_ylabel("10-fold CV MSE (training half)")
fig.tight_layout()
savefig(fig, 6, "lab-py-pcr-pls")


# =============================================================================================
# Applied exercises 8-11 of Section 6.6 (solutions shown in the lecture notes)
# =============================================================================================
def size_criteria(rss, tss, n, sigma2):
    """Cp (6.2), BIC as R's leaps computes it, and adjusted R^2 (6.4) for sizes 1..len(rss)."""
    rss = np.asarray(rss)
    d = np.arange(1, len(rss) + 1)
    return {"Cp": int(np.argmin(rss + 2 * d * sigma2)) + 1,
            "BIC": int(np.argmin(n * np.log(rss / tss) + (d + 1) * np.log(n))) + 1,
            "adjR2": int(np.argmax(1 - (rss / (n - d - 1)) / (tss / (n - 1)))) + 1}


def ols_coef(X, y, cols, labels):
    A = np.column_stack([np.ones(len(y))] + [X[:, j] for j in cols])
    b = np.linalg.lstsq(A, y, rcond=None)[0]
    return dict(zip(["(Intercept)"] + [labels[j] for j in cols], b.round(3)))


def cv_glmnet_lam(fit, X, y, lams, cv):
    """lambda with the smallest K-fold CV error (glmnet's lambda.min) and the one-SE choice."""
    def fp(Xtr, ytr, Xte, lam):
        b0, beta = fit(Xtr, ytr, lam)
        return b0 + Xte @ beta
    m, se = cv_lambda(fp, X, y, lams, cv)
    i = int(np.argmin(m))
    return lams[i], lams[np.nonzero(m <= m[i] + se[i])[0].min()]


# -- [E8] Exercise 8: simulated cubic; best subset, stepwise and the lasso on X, ..., X^10
step("E8", "Exercise 8")
rng8 = np.random.default_rng(8)
x8, eps = rng8.standard_normal(100), rng8.standard_normal(100)
y8 = 3 + 2 * x8 - 3 * x8 ** 2 + x8 ** 3 + eps            # beta0..beta3 = 3, 2, -3, 1
X8 = np.column_stack([x8 ** k for k in range(1, 11)])
lab8 = ["x%d" % k for k in range(1, 11)]
def selection_report(X, y):
    best, tss = best_subset(X, y)
    s2 = best[-1][1] / (len(y) - X.shape[1] - 1)
    fw, bw = forward_stepwise(X, y), backward_stepwise(X, y)
    for name, rs in [("exhaustive", [r for _, r in best]),
                     ("forward", [rss_of(X, y, fw[k]) for k in range(1, X.shape[1] + 1)]),
                     ("backward", [rss_of(X, y, bw[k]) for k in range(1, X.shape[1] + 1)])]:
        print(name, size_criteria(rs, tss, len(y), s2))
    return best


best8 = selection_report(X8, y8)
print(ols_coef(X8, y8, best8[2][0], lab8))
lams8 = 10 ** np.linspace(1, -3, 60)
lam8, _ = cv_glmnet_lam(lasso_glmnet, X8, y8, lams8, kfold)
b0, beta = lasso_glmnet(X8, y8, lam8)
print(round(lam8, 4), dict(zip(["(Intercept)"] + lab8, np.r_[b0, beta].round(3))))
y8b = 3 + 7 * x8 ** 7 + eps                              # part (f): only X^7 matters
best8b = selection_report(X8, y8b)
print(ols_coef(X8, y8b, best8b[0][0], lab8))
lam8b, _ = cv_glmnet_lam(lasso_glmnet, X8, y8b, 10 ** np.linspace(3, -2, 60), kfold)
b0, beta = lasso_glmnet(X8, y8b, lam8b)
print(dict(zip(["(Intercept)"] + lab8, np.r_[b0, beta].round(3))))

# -- [E9] Exercise 9: number of applications in the College data
step("E9", "Exercise 9")
College = isl_data("College", index_col=0)
yc9 = College["Apps"].to_numpy(float)
Xc9 = College.drop(columns="Apps").assign(Private=(College["Private"] == "Yes").astype(float))
Xc9 = Xc9.to_numpy(float)
tr9 = np.random.default_rng(9).permutation(len(yc9))[: len(yc9) // 2]
te9 = np.setdiff1d(np.arange(len(yc9)), tr9)
def mse9(pred):
    return np.mean((yc9[te9] - pred) ** 2)


res9 = {"LS": mse9(skl.LinearRegression().fit(Xc9[tr9], yc9[tr9]).predict(Xc9[te9]))}
for name, fit, lams in [("ridge", ridge_glmnet, 10 ** np.linspace(5, -1, 60)),
                        ("lasso", lasso_glmnet, 10 ** np.linspace(3.5, -1, 60))]:
    lam, _ = cv_glmnet_lam(fit, Xc9[tr9], yc9[tr9], lams, kfold)
    b0, beta = fit(Xc9[tr9], yc9[tr9], lam)
    res9[name] = mse9(b0 + Xc9[te9] @ beta)
print(int((beta != 0).sum()))                            # nonzero lasso coefficients
Ms9 = range(1, Xc9.shape[1] + 1)
M_pcr9 = int(np.argmin(cv_components(pcr, Xc9[tr9], yc9[tr9], kfold, Ms9)))
M_pls9 = int(np.argmin(cv_components(pls, Xc9[tr9], yc9[tr9], kfold, Ms9)))
res9["PCR"] = mse9(pcr(M_pcr9).fit(Xc9[tr9], yc9[tr9]).predict(Xc9[te9]))
res9["PLS"] = mse9(pls(M_pls9).fit(Xc9[tr9], yc9[tr9]).predict(Xc9[te9]).ravel())
print({"M.pcr": M_pcr9, "M.pls": M_pls9})
print({k: round(v) for k, v in res9.items()})
tss9 = np.mean((yc9[te9] - yc9[tr9].mean()) ** 2)
print({k: round(1 - v / tss9, 3) for k, v in res9.items()})        # test R^2

# -- [E10] Exercise 10: training and test error of best subset, p = 20, n = 1000
step("E10", "Exercise 10")
rng10 = np.random.default_rng(10)
X10 = rng10.standard_normal((1000, 20))
b10 = np.r_[2, -1.5, 1, 0.8, -0.6, 0.5, 0.4, -0.3, np.zeros(12)]    # 8 nonzero coefficients
y10 = X10 @ b10 + rng10.standard_normal(1000)
best10, _ = best_subset(X10[:100], y10[:100])            # 2^20 - 1 subsets of the 100 training rows
err10 = []
for cols, r in best10:
    A = np.column_stack([np.ones(100), X10[:100][:, cols]])
    coef = np.linalg.lstsq(A, y10[:100], rcond=None)[0]
    b = np.zeros(20)
    b[cols] = coef[1:]
    test = np.mean((y10[100:] - coef[0] - X10[100:][:, cols] @ coef[1:]) ** 2)
    err10.append((r / 100, test, np.sqrt(np.sum((b10 - b) ** 2))))
err10 = pd.DataFrame(err10, columns=["train", "test", "coef.err"], index=range(1, 21))
print(err10.loc[list(range(1, 11)) + [15, 20]].round(3))
print(err10["test"].idxmin(), err10["coef.err"].idxmin())

# -- [E11] Exercise 11: per capita crime rate in the Boston data, 10-fold CV
step("E11", "Exercise 11")
Boston = isl_data("Boston", index_col=0)
yb = Boston["crim"].to_numpy(float)
Xb = Boston.drop(columns="crim").to_numpy(float)
lab11 = list(Boston.columns.drop("crim"))
fold11 = np.random.default_rng(11).permutation(np.arange(len(yb)) % 10)
inner = skm.KFold(10, shuffle=True, random_state=1)
def cv11(fitpred):
    return np.mean([np.mean((yb[fold11 == j] - fitpred(Xb[fold11 != j], yb[fold11 != j],
                                                      Xb[fold11 == j])) ** 2) for j in range(10)])


def best_fp(k):
    def fp(X, y, Xn):
        cols = best_subset(X, y, max_size=k)[0][k - 1][0]
        return predict_subset(X, y, Xn, cols)
    return fp


def glm_fp(fit, lams):
    def fp(X, y, Xn):
        lam, _ = cv_glmnet_lam(fit, X, y, lams, inner)
        b0, beta = fit(X, y, lam)
        return b0 + Xn @ beta
    return fp


cv_best = np.array([cv11(best_fp(k)) for k in range(1, 13)])
print(cv_best.round(2))
res11 = {"null": cv11(lambda X, y, Xn: np.full(len(Xn), y.mean())),
         "LS": cv11(lambda X, y, Xn: skl.LinearRegression().fit(X, y).predict(Xn)),
         "best": cv_best.min(),
         "ridge": cv11(glm_fp(ridge_glmnet, 10 ** np.linspace(3, -2, 40))),
         "lasso": cv11(glm_fp(lasso_glmnet, 10 ** np.linspace(1, -3, 40)))}
cvp = cv_components(pcr, Xb, yb, skm.PredefinedSplit(fold11), range(1, 13))
res11["PCR"] = cvp.min()
print({k: round(v, 2) for k, v in res11.items()})
print(int(np.argmin(cv_best)) + 1)
_, lam_1se = cv_glmnet_lam(lasso_glmnet, Xb, yb, 10 ** np.linspace(1, -3, 40), inner)
b0, beta = lasso_glmnet(Xb, yb, lam_1se)
print(round(lam_1se, 3), dict(zip(["(Intercept)"] + lab11, np.r_[b0, beta].round(3))))
