# Chapter 6 lab: Linear Model Selection and Regularization (R version)
# Course: Applied Statistical Learning in R and Python, which follows James, Witten, Hastie and
# Tibshirani (2021), An Introduction to Statistical Learning with Applications in R, 2nd edition,
# Section 6.5, and the Python edition of 2023 (with Taylor).
#
# Predict a baseball player's Salary from the Hitters data with
#   [1] the data;  [2] best subset selection (leaps) and Cp, BIC, adjusted R^2;
#   [3] forward and backward stepwise selection;  [4] choosing the size by a validation set
#   and by 10-fold cross-validation;  [5] ridge regression (glmnet) at fixed lambda;
#   [6] ridge and the lasso on a training/test split with lambda chosen by cross-validation;
#   [7] principal components regression and [8] partial least squares (pls).
# The random seeds are those of the R edition, so the printed numbers can be compared with it.
# The Python version is ch06_lab.py; the two agree wherever the computation is deterministic.
#
# Run: Rscript ch06_lab.R      (prints the results; writes the R plots to ../img/ch06/lab-r-*.pdf)
# The code is my own.

suppressPackageStartupMessages({
  library(ISLR2)    # the Hitters data
  library(leaps)    # regsubsets(): best subset, forward and backward selection
  library(glmnet)   # ridge regression and the lasso
  library(pls)      # pcr() and plsr()
})
.this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
img_dir <- file.path(.this_dir, "..", "img", "ch06")
dir.create(img_dir, showWarnings = FALSE, recursive = TRUE)
step <- function(k, title) cat(sprintf("\n== [%s] %s\n", k, title))
options(width = 100, digits = 6)

# -- [1] The Hitters data: drop the players whose salary is missing
step(1, "The data")
dim(Hitters)
sum(is.na(Hitters$Salary))
Hitters <- na.omit(Hitters)
dim(Hitters)

# -- [2] Best subset selection: the best model of every size, and Cp, BIC, adjusted R^2
step(2, "Best subset selection")
regfit.full <- regsubsets(Salary ~ ., data = Hitters, nvmax = 19)
reg.summary <- summary(regfit.full)
round(reg.summary$rsq, 3)
c(adjr2 = which.max(reg.summary$adjr2), cp = which.min(reg.summary$cp),
  bic = which.min(reg.summary$bic))
coef(regfit.full, 6)

pdf(file.path(img_dir, "lab-r-criteria.pdf"), width = 8, height = 6)
par(mfrow = c(2, 2), mar = c(4.2, 4.2, 1, 1))
plot(reg.summary$rss, xlab = "Number of variables", ylab = "RSS", type = "l")
plot(reg.summary$adjr2, xlab = "Number of variables", ylab = "Adjusted RSq", type = "l")
points(11, reg.summary$adjr2[11], col = "red", cex = 2, pch = 20)
plot(reg.summary$cp, xlab = "Number of variables", ylab = "Cp", type = "l")
points(10, reg.summary$cp[10], col = "red", cex = 2, pch = 20)
plot(reg.summary$bic, xlab = "Number of variables", ylab = "BIC", type = "l")
points(6, reg.summary$bic[6], col = "red", cex = 2, pch = 20)
invisible(dev.off())
pdf(file.path(img_dir, "lab-r-bic-map.pdf"), width = 7, height = 5.5)
plot(regfit.full, scale = "bic")      # one row per model, black = variable included
invisible(dev.off())

# -- [3] Forward and backward stepwise selection
step(3, "Forward and backward stepwise selection")
regfit.fwd <- regsubsets(Salary ~ ., data = Hitters, nvmax = 19, method = "forward")
regfit.bwd <- regsubsets(Salary ~ ., data = Hitters, nvmax = 19, method = "backward")
round(coef(regfit.full, 7), 3)
round(coef(regfit.fwd, 7), 3)
round(coef(regfit.bwd, 7), 3)
# Cp along the forward path (the criterion the Python edition uses to stop forward selection)
fwd.summary <- summary(regfit.fwd)
which.min(fwd.summary$cp)
names(coef(regfit.fwd, which.min(fwd.summary$cp)))[-1]

# -- [4] Choosing the model size by a validation set and by 10-fold cross-validation
step(4, "Validation set and cross-validation")
set.seed(1)
train <- sample(c(TRUE, FALSE), nrow(Hitters), replace = TRUE)
test <- !train
regfit.best <- regsubsets(Salary ~ ., data = Hitters[train, ], nvmax = 19)
test.mat <- model.matrix(Salary ~ ., data = Hitters[test, ])
val.errors <- rep(NA, 19)
for (i in 1:19) {
  coefi <- coef(regfit.best, id = i)
  pred <- test.mat[, names(coefi)] %*% coefi
  val.errors[i] <- mean((Hitters$Salary[test] - pred)^2)
}
round(val.errors)
which.min(val.errors)

# regsubsets() has no predict() method, so we write one
predict.regsubsets <- function(object, newdata, id, ...) {
  form <- as.formula(object$call[[2]])
  mat <- model.matrix(form, newdata)
  coefi <- coef(object, id = id)
  mat[, names(coefi)] %*% coefi
}
k <- 10
n <- nrow(Hitters)
set.seed(1)
folds <- sample(rep(1:k, length = n))
cv.errors <- matrix(NA, k, 19, dimnames = list(NULL, paste(1:19)))
for (j in 1:k) {
  best.fit <- regsubsets(Salary ~ ., data = Hitters[folds != j, ], nvmax = 19)
  for (i in 1:19) {
    pred <- predict(best.fit, Hitters[folds == j, ], id = i)
    cv.errors[j, i] <- mean((Hitters$Salary[folds == j] - pred)^2)
  }
}
mean.cv.errors <- apply(cv.errors, 2, mean)
round(mean.cv.errors)
which.min(mean.cv.errors)
round(coef(regfit.full, 10), 3)   # the 10-variable model refitted on all the data
pdf(file.path(img_dir, "lab-r-cv-size.pdf"), width = 6, height = 4.5)
plot(mean.cv.errors, type = "b", xlab = "Number of variables", ylab = "10-fold CV error")
invisible(dev.off())

# -- [5] Ridge regression with glmnet: coefficients at fixed values of lambda
step(5, "Ridge regression at fixed lambda")
x <- model.matrix(Salary ~ ., Hitters)[, -1]   # 19 columns: the dummies LeagueN, DivisionW, NewLeagueN
y <- Hitters$Salary
grid <- 10^seq(10, -2, length = 100)
ridge.mod <- glmnet(x, y, alpha = 0, lambda = grid,   # alpha = 0: ridge; glmnet standardises x
                    control = list(thresh = 1e-12))  # tight convergence, so 3 decimals are exact
dim(coef(ridge.mod))
ridge.mod$lambda[50]
round(coef(ridge.mod)[, 50], 3)
sqrt(sum(coef(ridge.mod)[-1, 50]^2))
ridge.mod$lambda[60]
sqrt(sum(coef(ridge.mod)[-1, 60]^2))
# exact = TRUE refits at lambda = 50 instead of interpolating between grid values
round(predict(ridge.mod, s = 50, type = "coefficients", exact = TRUE, x = x, y = y)[1:20, ], 3)

# -- [6] Ridge and the lasso on a training/test split, lambda chosen by 10-fold CV
step(6, "Ridge and the lasso: test error")
set.seed(1)
train <- sample(1:nrow(x), nrow(x) / 2)
test <- (-train)
y.test <- y[test]
ridge.mod <- glmnet(x[train, ], y[train], alpha = 0, lambda = grid,
                    control = list(thresh = 1e-12))
ridge.pred <- predict(ridge.mod, s = 4, newx = x[test, ])
mean((ridge.pred - y.test)^2)                       # lambda = 4
mean((mean(y[train]) - y.test)^2)                   # intercept only
ridge.pred <- predict(ridge.mod, s = 0, newx = x[test, ], exact = TRUE, x = x[train, ], y = y[train])
mean((ridge.pred - y.test)^2)                       # lambda = 0: least squares
set.seed(1)
cv.out <- cv.glmnet(x[train, ], y[train], alpha = 0)
bestlam <- cv.out$lambda.min
bestlam
ridge.pred <- predict(ridge.mod, s = bestlam, newx = x[test, ])
mean((ridge.pred - y.test)^2)
out <- glmnet(x, y, alpha = 0)
round(predict(out, type = "coefficients", s = bestlam)[1:20, ], 2)
pdf(file.path(img_dir, "lab-r-cv-ridge.pdf"), width = 6, height = 4.5)
plot(cv.out)
invisible(dev.off())

lasso.mod <- glmnet(x[train, ], y[train], alpha = 1, lambda = grid)
set.seed(1)
cv.out <- cv.glmnet(x[train, ], y[train], alpha = 1)
bestlam <- cv.out$lambda.min
bestlam
lasso.pred <- predict(lasso.mod, s = bestlam, newx = x[test, ])
mean((lasso.pred - y.test)^2)
out <- glmnet(x, y, alpha = 1, lambda = grid)
lasso.coef <- predict(out, type = "coefficients", s = bestlam)[1:20, ]
round(lasso.coef[lasso.coef != 0], 2)
sum(lasso.coef[-1] != 0)
pdf(file.path(img_dir, "lab-r-lasso-path.pdf"), width = 6, height = 4.5)
plot(lasso.mod, xvar = "lambda")
invisible(dev.off())

# -- [7] Principal components regression
step(7, "Principal components regression")
set.seed(2)
pcr.fit <- pcr(Salary ~ ., data = Hitters, scale = TRUE, validation = "CV")
round(explvar(pcr.fit)[1:5], 2)                     # % variance of X per component
round(cumsum(explvar(pcr.fit))[1:5], 2)             # cumulative % of X
round(drop(R2(pcr.fit, estimate = "train", intercept = FALSE)$val)[1:5] * 100, 2)  # % of Salary
cv.rmsep <- drop(RMSEP(pcr.fit, estimate = "CV")$val)  # entries for 0, 1, ..., 19 components
round(cv.rmsep[1:5], 1)
which.min(cv.rmsep) - 1                             # number of components with the lowest CV error
set.seed(1)
pcr.fit <- pcr(Salary ~ ., data = Hitters, subset = train, scale = TRUE, validation = "CV")
which.min(drop(RMSEP(pcr.fit, estimate = "CV")$val)) - 1
pcr.pred <- predict(pcr.fit, x[test, ], ncomp = 5)
mean((pcr.pred - y.test)^2)
pdf(file.path(img_dir, "lab-r-pcr.pdf"), width = 6, height = 4.5)
validationplot(pcr.fit, val.type = "MSEP")
invisible(dev.off())

# -- [8] Partial least squares
step(8, "Partial least squares")
set.seed(1)
pls.fit <- plsr(Salary ~ ., data = Hitters, subset = train, scale = TRUE, validation = "CV")
round(drop(RMSEP(pls.fit, estimate = "CV")$val)[1:5], 1)
which.min(drop(RMSEP(pls.fit, estimate = "CV")$val)) - 1
pls.pred <- predict(pls.fit, x[test, ], ncomp = 1)
mean((pls.pred - y.test)^2)
pls.fit <- plsr(Salary ~ ., data = Hitters, scale = TRUE, ncomp = 1)
round(c(X = explvar(pls.fit)[1],
        Salary = 100 * drop(R2(pls.fit, estimate = "train", intercept = FALSE)$val)), 2)

# =============================================================================================
# Applied exercises 8-11 of Section 6.6 (solutions shown in the lecture notes)
# =============================================================================================

# -- [E8] Exercise 8: simulated cubic; best subset, stepwise and the lasso on X, ..., X^10
step("E8", "Exercise 8")
set.seed(1)
x8 <- rnorm(100)
eps <- rnorm(100)
y8 <- 3 + 2 * x8 - 3 * x8^2 + x8^3 + eps                # beta0..beta3 = 3, 2, -3, 1
d8 <- data.frame(y = y8, poly(x8, 10, raw = TRUE))
names(d8) <- c("y", paste0("x", 1:10))
for (m in c("exhaustive", "forward", "backward")) {
  s <- summary(regsubsets(y ~ ., data = d8, nvmax = 10, method = m))
  cat(m, ": Cp", which.min(s$cp), " BIC", which.min(s$bic), " adjR2", which.max(s$adjr2), "\n")
}
fit8 <- regsubsets(y ~ ., data = d8, nvmax = 10)
round(coef(fit8, 3), 3)
X8 <- as.matrix(d8[, -1])
set.seed(1)
cv8 <- cv.glmnet(X8, y8, alpha = 1)
round(coef(cv8, s = "lambda.min")[, 1], 3)
y8b <- 3 + 7 * x8^7 + eps                                # part (f): only X^7 matters
s <- summary(regsubsets(y ~ ., data = data.frame(y = y8b, X8), nvmax = 10))
c(Cp = which.min(s$cp), BIC = which.min(s$bic), adjR2 = which.max(s$adjr2))
round(coef(regsubsets(y ~ ., data = data.frame(y = y8b, X8), nvmax = 10), 1), 3)
set.seed(1)
cv8b <- cv.glmnet(X8, y8b, alpha = 1)
round(coef(cv8b, s = "lambda.min")[, 1], 3)

# -- [E9] Exercise 9: number of applications in the College data
step("E9", "Exercise 9")
set.seed(1)
tr9 <- sample(nrow(College), nrow(College) / 2)
Xc <- model.matrix(Apps ~ ., College)[, -1]
yc <- College$Apps
mse9 <- function(pred) mean((yc[-tr9] - pred)^2)
lm9 <- lm(Apps ~ ., data = College, subset = tr9)
res9 <- c(LS = mse9(predict(lm9, College[-tr9, ])))
for (a in c(0, 1)) {
  set.seed(1)
  cvf <- cv.glmnet(Xc[tr9, ], yc[tr9], alpha = a)
  res9[c("ridge", "lasso")[a + 1]] <- mse9(predict(cvf, Xc[-tr9, ], s = "lambda.min"))
}
sum(coef(cvf, s = "lambda.min")[-1] != 0)                 # nonzero lasso coefficients
set.seed(1)
pcr9 <- pcr(Apps ~ ., data = College, subset = tr9, scale = TRUE, validation = "CV")
M9 <- unname(which.min(drop(RMSEP(pcr9, estimate = "CV")$val)) - 1)
set.seed(1)
pls9 <- plsr(Apps ~ ., data = College, subset = tr9, scale = TRUE, validation = "CV")
P9 <- unname(which.min(drop(RMSEP(pls9, estimate = "CV")$val)) - 1)
res9["PCR"] <- mse9(predict(pcr9, College[-tr9, ], ncomp = M9))
res9["PLS"] <- mse9(predict(pls9, College[-tr9, ], ncomp = P9))
c(M.pcr = M9, M.pls = P9)
round(res9)
round(1 - res9 / mean((yc[-tr9] - mean(yc[tr9]))^2), 3)   # test R^2

# -- [E10] Exercise 10: training and test error of best subset, p = 20, n = 1000
step("E10", "Exercise 10")
set.seed(1)
X10 <- matrix(rnorm(1000 * 20), 1000, 20)
b10 <- c(2, -1.5, 1, 0.8, -0.6, 0.5, 0.4, -0.3, rep(0, 12))  # 8 nonzero coefficients
y10 <- drop(X10 %*% b10) + rnorm(1000)
d10 <- data.frame(y = y10, X10)
tr10 <- 1:100
fit10 <- regsubsets(y ~ ., data = d10[tr10, ], nvmax = 20)
Xte <- model.matrix(y ~ ., d10[-tr10, ])
err10 <- t(sapply(1:20, function(k) {
  cf <- coef(fit10, k)
  b <- setNames(rep(0, 20), paste0("X", 1:20))
  b[names(cf)[-1]] <- cf[-1]
  c(train = summary(fit10)$rss[k] / 100,
    test = mean((d10$y[-tr10] - Xte[, names(cf)] %*% cf)^2),
    coef.err = sqrt(sum((b10 - b)^2)))
}))
round(err10[c(1:10, 15, 20), ], 3)
which.min(err10[, "test"])
which.min(err10[, "coef.err"])

# -- [E11] Exercise 11: per capita crime rate in the Boston data, 10-fold CV
step("E11", "Exercise 11")
Xb <- model.matrix(crim ~ ., Boston)[, -1]
yb <- Boston$crim
set.seed(1)
fold11 <- sample(rep(1:10, length = nrow(Boston)))
cv11 <- function(fitpred) mean(sapply(1:10, function(j) {
  te <- fold11 == j
  mean((yb[te] - fitpred(Xb[!te, ], yb[!te], Xb[te, ]))^2)
}))
glm_fp <- function(a) function(X, y, Xn) {
  cvf <- cv.glmnet(X, y, alpha = a, nfolds = 10)
  predict(cvf, Xn, s = "lambda.min")
}
best_fp <- function(k) function(X, y, Xn) {
  f <- regsubsets(y ~ ., data = data.frame(y = y, X), nvmax = 12)
  cf <- coef(f, k)
  cbind(1, Xn)[, c(1, match(names(cf)[-1], colnames(Xn)) + 1)] %*% cf
}
set.seed(1)
cv.best <- sapply(1:12, function(k) cv11(best_fp(k)))
round(cv.best, 2)
set.seed(1)
res11 <- c(null = cv11(function(X, y, Xn) rep(mean(y), nrow(Xn))),
           LS = cv11(function(X, y, Xn) cbind(1, Xn) %*% coef(lm(y ~ X))),
           best = min(cv.best), ridge = cv11(glm_fp(0)), lasso = cv11(glm_fp(1)))
set.seed(1)
pcr11 <- pcr(crim ~ ., data = Boston, scale = TRUE, validation = "CV", segments = 10)
res11["PCR"] <- min(drop(MSEP(pcr11, estimate = "CV")$val))
round(res11, 2)
which.min(cv.best)
set.seed(1)
round(coef(cv.glmnet(Xb, yb, alpha = 1), s = "lambda.1se")[, 1], 3)
