# Chapter 7 lab: non-linear modelling (R version).
# Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie and
# Tibshirani (2021), An Introduction to Statistical Learning with Applications in R, 2nd edition,
# Section 7.8, and the Python lab of James et al. (2023), Section 7.8.
#
# The lab works through the Wage data (3000 men in the Mid-Atlantic region of the United States):
#   7.8.1  polynomial regression (orthogonal and raw bases, pointwise standard errors, nested F tests,
#          a polynomial logistic regression for wage > 250) and step functions;
#   7.8.2  regression splines (bs, ns), smoothing splines (smooth.spline with df = 16 and with
#          leave-one-out cross-validation) and local regression (loess);
#   7.8.3  generalised additive models: natural splines with lm(), smoothing splines and local
#          regression terms with gam(), F tests between nested GAMs, a logistic GAM.
# The Python version (ch07_lab.py) fits the same models; its own spline, local-regression and
# backfitting code is written out in full there. Run from the code/ folder:
#   Rscript ch07_lab.R
# Figures go to ../img/ch07/lab-r-*.svg and the key numbers to cache/ch07_lab_R.csv.

suppressPackageStartupMessages({
  library(ISLR2)
  library(splines)
  library(gam)
})
options(digits = 6, width = 100)

out_dir <- file.path("..", "img", "ch07")
dir.create(out_dir, showWarnings = FALSE, recursive = TRUE)
dir.create("cache", showWarnings = FALSE)
key <- list()                                 # key numbers, written to cache/ch07_lab_R.csv at the end
keep <- function(name, value) key[[name]] <<- unname(value)
open_svg <- function(name, width, height) {
  path <- file.path(out_dir, paste0(name, ".svg"))
  if (requireNamespace("svglite", quietly = TRUE)) svglite::svglite(path, width = width, height = height)
  else grDevices::svg(path, width = width, height = height)
}

# ================================================================== 7.8.1 polynomials and step functions
fit <- lm(wage ~ poly(age, 4), data = Wage)          # orthogonal polynomial basis
print(coef(summary(fit)))

fit2 <- lm(wage ~ poly(age, 4, raw = TRUE), data = Wage)   # the raw powers age, age^2, age^3, age^4
print(coef(summary(fit2)))
fit2a <- lm(wage ~ age + I(age^2) + I(age^3) + I(age^4), data = Wage)
fit2b <- lm(wage ~ cbind(age, age^2, age^3, age^4), data = Wage)
stopifnot(isTRUE(all.equal(coef(fit2a), coef(fit2b), check.attributes = FALSE)))

agelims <- range(Wage$age)
age.grid <- seq(from = agelims[1], to = agelims[2])
preds <- predict(fit, newdata = list(age = age.grid), se = TRUE)
se.bands <- cbind(preds$fit + 2 * preds$se.fit, preds$fit - 2 * preds$se.fit)
preds2 <- predict(fit2, newdata = list(age = age.grid), se = TRUE)
cat("largest difference between the two bases' fitted values:", max(abs(preds$fit - preds2$fit)), "\n")

# the standard error by hand: Var f(x0) = l0' C l0 with C the covariance matrix of the coefficients
L0 <- cbind(1, poly(age.grid, 4, raw = TRUE))
se.hand <- sqrt(rowSums((L0 %*% vcov(fit2)) * L0))
cat("largest difference between predict()'s and the hand-computed standard errors:",
    max(abs(se.hand - preds2$se.fit)), "\n")
keep("poly4_coef", coef(fit)); keep("poly4_se1", coef(summary(fit))[2, 2])
keep("poly4_fit_age50", preds$fit[age.grid == 50]); keep("poly4_se_age50", preds$se.fit[age.grid == 50])
keep("poly4_se_age80", preds$se.fit[age.grid == 80])

# nested F tests for the degree
fit.1 <- lm(wage ~ age, data = Wage)
fit.2 <- lm(wage ~ poly(age, 2), data = Wage)
fit.3 <- lm(wage ~ poly(age, 3), data = Wage)
fit.4 <- lm(wage ~ poly(age, 4), data = Wage)
fit.5 <- lm(wage ~ poly(age, 5), data = Wage)
av <- anova(fit.1, fit.2, fit.3, fit.4, fit.5)
print(av)
print(coef(summary(fit.5)))
cat("t^2 for the quadratic term:", coef(summary(fit.5))[3, 3]^2, "\n")
keep("anova_rss", av$RSS); keep("anova_F", av$F[-1]); keep("anova_p", av$`Pr(>F)`[-1])

fit.1 <- lm(wage ~ education + age, data = Wage)
fit.2 <- lm(wage ~ education + poly(age, 2), data = Wage)
fit.3 <- lm(wage ~ education + poly(age, 3), data = Wage)
av2 <- anova(fit.1, fit.2, fit.3)
print(av2)
keep("anova_edu_F", av2$F[-1])

# polynomial logistic regression for the event wage > 250
fit <- glm(I(wage > 250) ~ poly(age, 4), data = Wage, family = binomial)
print(coef(summary(fit)))
preds <- predict(fit, newdata = list(age = age.grid), se = TRUE)        # on the logit scale
pfit <- exp(preds$fit) / (1 + exp(preds$fit))
se.bands.logit <- cbind(preds$fit + 2 * preds$se.fit, preds$fit - 2 * preds$se.fit)
se.bands.p <- exp(se.bands.logit) / (1 + exp(se.bands.logit))
keep("logit_coef", coef(fit)); keep("logit_p_age50", pfit[age.grid == 50])
keep("logit_band_age80", se.bands.p[age.grid == 80, ])

open_svg("lab-r-poly", 9, 4.2)
par(mfrow = c(1, 2), mar = c(4.5, 4.5, 1, 1), oma = c(0, 0, 3, 0))
plot(Wage$age, Wage$wage, xlim = agelims, cex = .5, col = "darkgrey", xlab = "age", ylab = "wage")
title("Degree-4 polynomial", outer = TRUE)
lines(age.grid, predict(lm(wage ~ poly(age, 4), data = Wage), list(age = age.grid)), lwd = 2, col = "blue")
matlines(age.grid, se.bands, lwd = 1, col = "blue", lty = 3)
plot(Wage$age, I(Wage$wage > 250), xlim = agelims, type = "n", ylim = c(0, .2),
     xlab = "age", ylab = "Pr(wage > 250 | age)")
set.seed(1)
points(jitter(Wage$age), I((Wage$wage > 250) / 5), cex = .5, pch = "|", col = "darkgrey")
lines(age.grid, pfit, lwd = 2, col = "blue")
matlines(age.grid, se.bands.p, lwd = 1, col = "blue", lty = 3)
invisible(dev.off())

# step functions: cut() makes four bins of equal width
print(table(cut(Wage$age, 4)))
fit <- lm(wage ~ cut(age, 4), data = Wage)
print(coef(summary(fit)))
keep("step_coef", coef(fit))

# ================================================================== 7.8.2 splines
fit <- lm(wage ~ bs(age, knots = c(25, 40, 60)), data = Wage)   # cubic spline, three knots
pred <- predict(fit, newdata = list(age = age.grid), se = TRUE)
print(dim(bs(Wage$age, knots = c(25, 40, 60))))
print(dim(bs(Wage$age, df = 6)))
print(attr(bs(Wage$age, df = 6), "knots"))
keep("bs_coef", coef(fit)); keep("bs_fit_age50", pred$fit[age.grid == 50])
keep("bs_se_age80", pred$se.fit[age.grid == 80])

fit2 <- lm(wage ~ ns(age, df = 4), data = Wage)                # natural spline, 4 degrees of freedom
pred2 <- predict(fit2, newdata = list(age = age.grid), se = TRUE)
print(attr(ns(Wage$age, df = 4), "knots"))
keep("ns4_coef", coef(fit2)); keep("ns4_se_age80", pred2$se.fit[age.grid == 80])
fit5 <- lm(wage ~ ns(age, df = 5), data = Wage)                # the Python lab's choice
print(coef(summary(fit5)))
keep("ns5_coef", coef(fit5))

open_svg("lab-r-splines", 9, 4.2)
par(mfrow = c(1, 2), mar = c(4.5, 4.5, 2, 1))
plot(Wage$age, Wage$wage, col = "gray", cex = .5, xlab = "age", ylab = "wage", main = "Regression splines")
lines(age.grid, pred$fit, lwd = 2)
lines(age.grid, pred$fit + 2 * pred$se, lty = "dashed")
lines(age.grid, pred$fit - 2 * pred$se, lty = "dashed")
lines(age.grid, pred2$fit, col = "red", lwd = 2)
legend("topright", legend = c("bs, knots 25, 40, 60", "ns, df = 4"), col = c("black", "red"), lwd = 2, cex = .8)

# smoothing splines: fixed df, and lambda chosen by leave-one-out cross-validation
plot(Wage$age, Wage$wage, xlim = agelims, cex = .5, col = "darkgrey", xlab = "age", ylab = "wage")
title("Smoothing spline")
fit <- smooth.spline(Wage$age, Wage$wage, df = 16)
fit2 <- suppressWarnings(smooth.spline(Wage$age, Wage$wage, cv = TRUE))   # warns about tied ages
print(fit2$df)
lines(fit, col = "red", lwd = 2)
lines(fit2, col = "blue", lwd = 2)
legend("topright", legend = c("16 DF", sprintf("%.1f DF", fit2$df)), col = c("red", "blue"), lty = 1, lwd = 2, cex = .8)
invisible(dev.off())
keep("ss16_df", fit$df); keep("ss16_lambda", fit$lambda)
keep("ss16_fit", predict(fit, c(20, 40, 60, 80))$y)
keep("sscv_df", fit2$df); keep("sscv_cv", fit2$cv.crit); keep("sscv_fit", predict(fit2, c(20, 40, 60, 80))$y)
# all.knots = TRUE puts a knot at each of the 61 distinct ages: the exact smoothing spline of (7.11)
fit3 <- smooth.spline(Wage$age, Wage$wage, df = 16, all.knots = TRUE)
keep("ss16all_fit", predict(fit3, c(20, 40, 60, 80))$y)

# local regression (loess fits local quadratics by default: degree = 2)
fit <- loess(wage ~ age, span = .2, data = Wage)
fit2 <- loess(wage ~ age, span = .5, data = Wage)
open_svg("lab-r-loess", 4.6, 4.2)
par(mar = c(4.5, 4.5, 2, 1))
plot(Wage$age, Wage$wage, xlim = agelims, cex = .5, col = "darkgrey", xlab = "age", ylab = "wage")
title("Local regression")
lines(age.grid, predict(fit, data.frame(age = age.grid)), col = "red", lwd = 2)
lines(age.grid, predict(fit2, data.frame(age = age.grid)), col = "blue", lwd = 2)
legend("topright", legend = c("Span = 0.2", "Span = 0.5"), col = c("red", "blue"), lty = 1, lwd = 2, cex = .8)
invisible(dev.off())
keep("lo2_fit", predict(fit, data.frame(age = c(20, 40, 60, 80))))
keep("lo2_enp", fit$enp)
# the same fits computed exactly at every point (no interpolation), and local linear fits
direct <- loess.control(surface = "direct")
keep("lo2d_fit", predict(loess(wage ~ age, span = .2, data = Wage, control = direct), data.frame(age = c(20, 40, 60, 80))))
keep("lo5d_fit", predict(loess(wage ~ age, span = .5, data = Wage, control = direct), data.frame(age = c(20, 40, 60, 80))))
keep("lo2d1_fit", predict(loess(wage ~ age, span = .2, degree = 1, data = Wage, control = direct), data.frame(age = c(20, 40, 60, 80))))

# ================================================================== 7.8.3 generalised additive models
gam1 <- lm(wage ~ ns(year, 4) + ns(age, 5) + education, data = Wage)     # natural splines: plain least squares
keep("gam1_rss", sum(resid(gam1)^2))

gam.m3 <- gam(wage ~ s(year, 4) + s(age, 5) + education, data = Wage)    # smoothing splines: backfitting
open_svg("lab-r-gam", 10, 3.6)
par(mfrow = c(1, 3), mar = c(4.5, 4.5, 1, 1))
plot(gam.m3, se = TRUE, col = "blue")
invisible(dev.off())
open_svg("lab-r-gam-ns", 10, 3.6)
par(mfrow = c(1, 3), mar = c(4.5, 4.5, 1, 1))
plot.Gam(gam1, se = TRUE, col = "red")
invisible(dev.off())

gam.m1 <- gam(wage ~ s(age, 5) + education, data = Wage)
gam.m2 <- gam(wage ~ year + s(age, 5) + education, data = Wage)
av3 <- anova(gam.m1, gam.m2, gam.m3, test = "F")
print(av3)
print(summary(gam.m3))
keep("gam_dev", c(deviance(gam.m1), deviance(gam.m2), deviance(gam.m3)))
keep("gam_resdf", av3$`Resid. Df`); keep("gam_F", av3$F[-1]); keep("gam_p", av3$`Pr(>F)`[-1])
keep("gam_m3_fit", fitted(gam.m3)[1:5])

preds <- predict(gam.m2, newdata = Wage)
keep("gam_m2_pred", preds[1:5])

gam.lo <- gam(wage ~ s(year, df = 4) + lo(age, span = 0.7) + education, data = Wage)
keep("gam_lo_dev", deviance(gam.lo))
gam.lo.i <- suppressWarnings(gam(wage ~ lo(year, age, span = 0.5) + education, data = Wage))  # warnings from lowesd's workspace
keep("gam_loi_dev", deviance(gam.lo.i))

# logistic GAM for wage > 250
gam.lr <- gam(I(wage > 250) ~ year + s(age, df = 5) + education, family = binomial, data = Wage)
print(table(Wage$education, I(Wage$wage > 250)))
gam.lr.s <- gam(I(wage > 250) ~ year + s(age, df = 5) + education, family = binomial,
                data = Wage, subset = (education != "1. < HS Grad"))
open_svg("lab-r-gam-logistic", 10, 3.6)
par(mfrow = c(1, 3), mar = c(4.5, 4.5, 1, 1))
plot(gam.lr.s, se = TRUE, col = "green")
invisible(dev.off())
keep("gamlr_dev", deviance(gam.lr)); keep("gamlrs_dev", deviance(gam.lr.s))
keep("gamlrs_coef", coef(gam.lr.s))

# ================================================================== applied exercises (Section 7.9)
# Cross-validation folds are fixed by row number (row i is in fold i mod 10), so that the R and the
# Python solutions use exactly the same folds and give the same numbers.
fold_id <- function(n, K = 10) (seq_len(n) - 1) %% K
cv_lm <- function(formula, data, K = 10) {
  fold <- fold_id(nrow(data), K); sse <- 0
  for (k in 0:(K - 1)) {
    fit <- lm(formula, data = data[fold != k, ])
    sse <- sse + sum((data[fold == k, all.vars(formula)[1]] - predict(fit, data[fold == k, ]))^2)
  }
  sse / nrow(data)
}

# Exercise 6(a): the degree of a polynomial in age, by CV and by F tests
cv_poly <- sapply(1:10, function(d) cv_lm(as.formula(sprintf("wage ~ poly(age, %d)", d)), Wage))
print(round(cv_poly, 2)); cat("Exercise 6(a): CV chooses degree", which.min(cv_poly), "\n")
keep("ex6_cv_poly", cv_poly)
# Exercise 6(b): the number of equal-width bins. The breaks must come from the whole range of age:
# cut() inside a formula would recompute them on each held-out fold.
cv_cut <- sapply(2:10, function(k) {
  br <- seq(min(Wage$age), max(Wage$age), length.out = k + 1); br[1] <- br[1] - 1; br[k + 1] <- br[k + 1] + 1
  d <- data.frame(wage = Wage$wage, bin = cut(Wage$age, br))
  cv_lm(wage ~ bin, d)
})
print(round(cv_cut, 2)); cat("Exercise 6(b): CV chooses", which.min(cv_cut) + 1, "bins\n")
keep("ex6_cv_cut", cv_cut)

# Exercise 7: marital status and job class, added to an additive model
g0 <- gam(wage ~ s(age, 5) + education, data = Wage)
g1 <- gam(wage ~ s(age, 5) + education + jobclass, data = Wage)
g2 <- gam(wage ~ s(age, 5) + education + jobclass + maritl, data = Wage)
av7 <- anova(g0, g1, g2, test = "F"); print(av7)
print(round(coef(g2)[c("jobclass2. Information", grep("maritl", names(coef(g2)), value = TRUE))], 3))
keep("ex7_dev", c(deviance(g0), deviance(g1), deviance(g2))); keep("ex7_F", av7$F[-1])

# Exercise 8: Auto, mpg against horsepower and an additive model
cv_auto <- sapply(1:6, function(d) cv_lm(as.formula(sprintf("mpg ~ poly(horsepower, %d)", d)), Auto))
print(round(cv_auto, 3)); keep("ex8_cv_poly", cv_auto)
a_lin <- gam(mpg ~ horsepower + weight + year + factor(origin), data = Auto)
a_gam <- gam(mpg ~ s(horsepower, 4) + s(weight, 4) + s(year, 4) + factor(origin), data = Auto)
av8 <- anova(a_lin, a_gam, test = "F"); print(av8)
keep("ex8_dev", c(deviance(a_lin), deviance(a_gam))); keep("ex8_F", av8$F[2])

# Exercise 9: Boston, nox against dis
b3 <- lm(nox ~ poly(dis, 3), data = Boston); print(coef(summary(b3)))
keep("ex9_cubic_coef", coef(b3))
rss_poly <- sapply(1:10, function(d) sum(resid(lm(nox ~ poly(dis, d), data = Boston))^2))
print(round(rss_poly, 4)); keep("ex9_rss_poly", rss_poly)
cv_bpoly <- sapply(1:10, function(d) cv_lm(as.formula(sprintf("nox ~ poly(dis, %d)", d)), Boston))
cat("Exercise 9(c): CV chooses degree", which.min(cv_bpoly), "\n"); keep("ex9_cv_poly", cv_bpoly)
b4 <- lm(nox ~ bs(dis, df = 4), data = Boston); print(attr(bs(Boston$dis, df = 4), "knots"))
print(coef(summary(b4))); keep("ex9_bs4_coef", coef(b4))
rss_bs <- sapply(3:16, function(d) sum(resid(lm(nox ~ bs(dis, df = d), data = Boston))^2))
print(round(rss_bs, 4)); keep("ex9_rss_bs", rss_bs)
cv_bbs <- suppressWarnings(sapply(3:16, function(d) cv_lm(as.formula(sprintf("nox ~ bs(dis, df = %d)", d)), Boston)))
cat("Exercise 9(f): CV chooses df", which.min(cv_bbs) + 2, "\n"); keep("ex9_cv_bs", cv_bbs)

# Exercise 10: College, forward stepwise selection then an additive model
train <- fold_id(nrow(College), 2) == 0          # alternate rows: 389 training, 388 test colleges
fwd <- leaps::regsubsets(Outstate ~ ., data = College[train, ], nvmax = 17, method = "forward")
bic <- summary(fwd)$bic; size <- which.min(bic)
vars <- sub("PrivateYes", "Private", names(coef(fwd, size))[-1]); print(vars)
keep("ex10_bic", bic)
num <- setdiff(vars, "Private")                  # the selected quantitative inputs get smoothing splines
lin10 <- lm(reformulate(vars, "Outstate"), data = College[train, ])
gam10 <- gam(reformulate(c("Private", sprintf("s(%s, 4)", num)), "Outstate"), data = College[train, ])
test_mse <- function(f) mean((College$Outstate[!train] - predict(f, College[!train, ]))^2)
cat("Exercise 10: test MSE linear", test_mse(lin10), " GAM", test_mse(gam10), "\n")
keep("ex10_test", c(test_mse(lin10), test_mse(gam10))); keep("ex10_gam_dev", deviance(gam10))
nl_p <- sapply(num, function(v) {
  f0 <- update(gam10, as.formula(sprintf(". ~ . - s(%s, 4) + %s", v, v)))
  anova(f0, gam10, test = "F")$`Pr(>F)`[2]
})
print(signif(nl_p, 3)); keep("ex10_nl_p", nl_p)

# Exercise 11: backfitting for multiple regression with two inputs (deterministic toy data)
i <- 1:100
x1 <- sin(i); x2 <- 0.6 * x1 + 0.8 * cos(2.7 * i); y11 <- 1 + 2 * x1 - 1.5 * x2 + 0.5 * sin(17.3 * i^1.5)
beta1 <- 10; path <- matrix(NA, 1000, 3)
for (it in 1:1000) {
  a <- y11 - beta1 * x1; c2 <- coef(lm(a ~ x2)); beta2 <- c2[2]
  a <- y11 - beta2 * x2; c1 <- coef(lm(a ~ x1)); beta1 <- c1[2]
  path[it, ] <- c(c1[1], beta1, beta2)
}
ls11 <- coef(lm(y11 ~ x1 + x2)); print(rbind(backfitting = path[1000, ], least.squares = ls11))
err11 <- apply(abs(sweep(path, 2, ls11)), 1, max)
cat("Exercise 11: iterations until within 1e-6 of least squares:", which(err11 < 1e-6)[1], "\n")
keep("ex11_ls", ls11); keep("ex11_iter", which(err11 < 1e-6)[1]); keep("ex11_first", path[1, ])

# Exercise 12: backfitting with p = 100 inputs
n12 <- 1000; p12 <- 100
X12 <- outer(1:n12, 1:p12, function(i, j) sin(0.7 * i * j + 1.3 * j + 0.11 * i))
beta12 <- ((-1)^(1:p12)) * (1:p12) / 50
y12 <- drop(X12 %*% beta12) + 0.5 * sin(5.7 * (1:n12))
ls12 <- coef(lm(y12 ~ X12))
b <- rep(0, p12); b0 <- 0; err12 <- numeric(50)
for (it in 1:50) {
  for (j in 1:p12) {
    r <- y12 - drop(X12[, -j] %*% b[-j])
    cf <- coef(lm(r ~ X12[, j])); b0 <- cf[1]; b[j] <- cf[2]
  }
  err12[it] <- max(abs(c(b0, b) - ls12))
}
print(signif(err12[1:10], 3))
cat("Exercise 12: sweeps until within 1e-6:", which(err12 < 1e-6)[1], "\n")
keep("ex12_err", err12[1:10]); keep("ex12_iter", which(err12 < 1e-6)[1])

# ================================================================== key numbers
flat <- unlist(lapply(names(key), function(k) {
  v <- key[[k]]; nm <- if (length(v) > 1) paste0(k, "_", seq_along(v)) else k
  setNames(as.numeric(v), nm)
}))
write.csv(data.frame(name = names(flat), value = sprintf("%.10g", flat)), "cache/ch07_lab_R.csv", row.names = FALSE)
cat("wrote", length(flat), "numbers to cache/ch07_lab_R.csv\n")
