# Chapter 12. Supervised Learning
# Course: Statistical Methods for Modern Biology (following Holmes and Huber 2019,
# Modern Statistics for Modern Biology, Cambridge University Press, Chapter 12).
#
# Reproduces every figure of the chapter that is computed from data or by simulation:
#   Figure 12.1   overfitting: two loess fits with different spans to simulated data
#   Figure 12.3   densities of the five clinical variables of the diabetes data, by group
#   Figure 12.5   scatterplot of sspg and glucose, coloured by diabetes group
#   Figure 12.6   LDA classification regions, class centres and the pooled-covariance ellipses
#   Figure 12.7   the same with uniform class priors
#   Figure 12.8   pairs plot of Fn1, Timd2, Gata4, Sox7 in the wild-type mouse embryo cells
#   Figure 12.9   LDA classification regions in the plane of the two discriminant coordinates
#   Figure 12.10  QDA classification regions for all six pairs of the four genes
#   Figure 12.11  training misclassification rate of LDA on random data, n = 20, p = 2..21
#   Figure 12.12  training and leave-one-out misclassification rates on one random data set
#   Figure 12.13  the same averaged over 100 data sets in which 6 features carry signal
#   Figure 12.15  side length of a sub-cube holding 10 of 10^6 uniform points, against dimension
#   Figure 12.16  fraction of the unit cube within 0.01 of its surface, against dimension
#   Figure 12.17  coefficient of variation of the distance between two uniform points
#   Figure 12.19  histograms of two microbiome features (Zeller et al. 2014)
#   Figure 12.20  lasso logistic regression paths on the microbiome data
#   Figure 12.21  cross-validated binomial deviance against log(lambda)
#   Figure 12.22  the same after an asinh transformation of the features
#   Figure 12.23  cross-validated misclassification error, mouse E3.25 cells, genotype
#   Figure 12.24  histogram of per-gene t-test p-values, E3.25 wild type versus FGF4-KO
#   Figure 12.25  cross-validated accuracy of a neural network over its tuning grid (caret)
#
# Skipped (not computed from data):
#   Figure 12.2   the roles of X and Y in training and test data (schematic)
#   Figure 12.4   nuclear morphology classes with example images from Neumann et al. (2010)
#                 (microscopy images and a bar chart taken from the paper)
#   Figure 12.14  idealised training and test error against model complexity (schematic)
#   Figure 12.18  two bull's-eye targets illustrating bias and variance (hand-drawn schematic)
#
# It also writes the small JSON files used by the browser demonstrations:
#   data/ch12-diabetes.json   the 145 patients of the diabetes data (5 variables and the group)
#   data/ch12-cv.json         three simulated data sets: pure noise (Figure 12.12), six informative
#                             features (one replicate of Figure 12.13), and 500 noise features for the
#                             feature-selection experiment, with the R error curves
#   data/ch12-zeller.json     260 species-level abundances (prevalence >= 10%) of the 114 samples,
#                             the class labels and a fixed 10-fold assignment
# and cache/ch12-results.rds with the numbers quoted in the lecture notes.
#
# Run: Rscript ch12_supervised_learning.R              (all figures and data)
#      Rscript ch12_supervised_learning.R fig_12_13    (one figure)
# The code is my own; it follows the analysis steps of the book's Chapter 12.

.this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
source(file.path(.this_dir, "common.R"))

suppressPackageStartupMessages({
  library(MASS)
  library(dplyr)
  library(tidyr)
  library(glmnet)
})

CH <- 12
CACHE <- file.path(CODE_DIR, "cache")
dir.create(CACHE, showWarnings = FALSE)
cache_path <- function(name) file.path(CACHE, paste0("ch12-", name, ".rds"))
cached <- function(name, expr) {
  f <- cache_path(name)
  if (file.exists(f)) return(readRDS(f))
  val <- force(expr)
  saveRDS(val, f)
  val
}
GROUP_COL <- c(unname(COL["blue"]), unname(COL["orange"]), unname(COL["green"]))

# =============================================================================================
# data

load_diabetes <- function() {
  e <- new.env()
  data("diabetes", package = "rrcov", envir = e)
  e$diabetes
}

# the Hiiragi2013 ExpressionSet 'x' (101 arrays, 45101 probes)
load_hiiragi <- function() {
  suppressPackageStartupMessages(library(Hiiragi2013))
  e <- new.env()
  data("x", package = "Hiiragi2013", envir = e)
  e$x
}

# four probes for Fn1, Timd2, Gata4, Sox7 in the wild-type cells
EMBRYO_PROBES <- c("1426642_at", "1418765_at", "1418864_at", "1416564_at")
load_embryo <- function() {
  cached("embryo", {
    x <- load_hiiragi()
    sym <- tryCatch({
      suppressPackageStartupMessages(library(mouse4302.db))
      a <- suppressMessages(AnnotationDbi::select(mouse4302.db, keys = EMBRYO_PROBES,
                                                  columns = c("SYMBOL", "GENENAME")))
      setNames(a$SYMBOL, a$PROBEID)[EMBRYO_PROBES]
    }, error = function(e) setNames(c("Fn1", "Timd2", "Gata4", "Sox7"), EMBRYO_PROBES))
    wt <- x$genotype == "WT"
    d <- as.data.frame(t(Biobase::exprs(x)[EMBRYO_PROBES, wt]))
    colnames(d) <- unname(sym)
    d$Embryonic.day <- droplevels(factor(x$Embryonic.day[wt]))
    rownames(d) <- NULL
    d
  })
}

# Zeller et al. (2014): normal and cancer samples of ExperimentHub record EH361
load_zeller <- function() {
  cached("zeller", {
    suppressPackageStartupMessages(library(ExperimentHub))
    eh <- ExperimentHub()
    z <- eh[["EH361"]]
    zn <- z[, z$disease %in% c("n", "cancer")]
    list(X = t(Biobase::exprs(zn)), disease = factor(zn$disease), pdata = Biobase::pData(zn))
  })
}

# species-level features (last taxonomic rank s__) present in at least 10% of the samples,
# rounded to 4 significant digits: the data set of the elastic-net demonstration
zeller_species <- function(z = load_zeller()) {
  lev <- vapply(strsplit(colnames(z$X), "|", fixed = TRUE), function(s) substr(s[length(s)], 1, 1), "")
  keep <- lev == "s" & colMeans(z$X > 0) >= 0.1
  X <- signif(z$X[, keep], 4)
  short <- sub("^s__", "", vapply(strsplit(colnames(X), "|", fixed = TRUE), function(s) s[length(s)], ""))
  list(X = X, y = z$disease, short = short)
}

short_taxon <- function(nm) {
  s <- strsplit(nm, "|", fixed = TRUE)[[1]]
  sub("^[a-z]__", "", s[length(s)])
}

# =============================================================================================
# helpers

make1Dgrid <- function(x, n = 100) {
  rg <- grDevices::extendrange(x)
  seq(rg[1], rg[2], length.out = n)
}

# Linear discriminant analysis written out step by step as MASS::lda (method "moment") does it:
# sphere with the pooled within-class covariance (divisor n - K) through an SVD that drops
# directions with singular value below tol, then rotate to the between-class principal axes.
# The browser demonstrations implement exactly these steps.
lda_mass <- function(x, g, prior = NULL, tol = 1e-4) {
  x <- as.matrix(x); g <- factor(g); g <- droplevels(g)
  n <- nrow(x); p <- ncol(x); K <- nlevels(g)
  counts <- tabulate(as.integer(g), K)
  if (is.null(prior)) prior <- counts / n
  means <- rowsum(x, g) / counts
  R <- x - means[as.integer(g), , drop = FALSE]
  f1 <- sqrt(colSums(R^2) / (n - 1))
  S0 <- diag(1 / f1, p)
  sv <- svd(sqrt(1 / (n - K)) * R %*% S0, nu = 0)
  rank <- sum(sv$d > tol)
  scaling <- S0 %*% sv$v[, seq_len(rank), drop = FALSE] %*% diag(1 / sv$d[seq_len(rank)], rank)
  xbar <- colSums(prior * means)
  X2 <- sqrt((n * prior) / (K - 1)) * sweep(means, 2, xbar) %*% scaling
  sv2 <- svd(X2, nu = 0)
  rank2 <- sum(sv2$d > tol * sv2$d[1])
  scaling <- scaling %*% sv2$v[, seq_len(rank2), drop = FALSE]
  list(prior = prior, means = means, scaling = scaling, lev = levels(g), rank = rank, svd = sv2$d[seq_len(rank2)])
}
lda_mass_predict <- function(fit, newx) {
  newx <- as.matrix(newx)
  m <- colSums(fit$prior * fit$means)
  z <- sweep(newx, 2, m) %*% fit$scaling
  dm <- sweep(fit$means, 2, m) %*% fit$scaling
  dist <- matrix(0.5 * rowSums(dm^2) - log(fit$prior), nrow(z), length(fit$prior), byrow = TRUE) - z %*% t(dm)
  post <- exp(-(dist - apply(dist, 1, min)))
  post <- post / rowSums(post)
  list(class = factor(fit$lev[max.col(-dist, ties.method = "first")], levels = fit$lev), posterior = post, x = z)
}

# leave-one-out and training misclassification rates of MASS::lda on (x, resp)
loo_lda <- function(x, resp) {
  r <- vapply(seq_len(nrow(x)), function(i) {
    fit <- suppressWarnings(lda(x[-i, , drop = FALSE], resp[-i]))
    ptrn <- predict(fit, newdata = x[-i, , drop = FALSE])$class
    ptst <- predict(fit, newdata = x[i, , drop = FALSE])$class
    c(train = mean(ptrn != resp[-i]), test = as.numeric(ptst != resp[i]))
  }, numeric(2))
  rowMeans(r)
}

# per-feature two-sample t statistics (equal variances), rows = samples
tstats <- function(x, g) {
  g <- factor(g); a <- g == levels(g)[1]
  n1 <- sum(a); n2 <- sum(!a)
  m1 <- colMeans(x[a, , drop = FALSE]); m2 <- colMeans(x[!a, , drop = FALSE])
  v <- (colSums(sweep(x[a, , drop = FALSE], 2, m1)^2) + colSums(sweep(x[!a, , drop = FALSE], 2, m2)^2)) / (n1 + n2 - 2)
  (m1 - m2) / sqrt(v * (1 / n1 + 1 / n2))
}

# leave-one-out LDA error after selecting the k features with largest |t|, either once on all data
# ("outside" the loop: the wrong way) or afresh in every training set ("inside": the right way)
selection_cv <- function(x, resp, k) {
  top <- function(xx, gg) order(-abs(tstats(xx, gg)), seq_len(ncol(xx)))[seq_len(k)]
  sel_all <- top(x, resp)
  err <- vapply(seq_len(nrow(x)), function(i) {
    f1 <- suppressWarnings(lda(x[-i, sel_all, drop = FALSE], resp[-i]))
    s_in <- top(x[-i, , drop = FALSE], resp[-i])
    f2 <- suppressWarnings(lda(x[-i, s_in, drop = FALSE], resp[-i]))
    c(outside = as.numeric(predict(f1, x[i, sel_all, drop = FALSE])$class != resp[i]),
      inside = as.numeric(predict(f2, x[i, s_in, drop = FALSE])$class != resp[i]))
  }, numeric(2))
  rowMeans(err)
}

# the Mahalanobis unit circle of the pooled within-class covariance W around a centre
pooled_cov <- function(x, g) {
  g <- factor(g); means <- rowsum(as.matrix(x), g) / as.vector(table(g))
  R <- as.matrix(x) - means[as.integer(g), , drop = FALSE]
  crossprod(R) / (nrow(R) - nlevels(g))
}
ellipse_pts <- function(centre, W, r = 1, n = 360) {
  t <- seq(0, 2 * pi, length.out = n)
  U <- cbind(cos(t), sin(t)) * r
  sweep(U %*% chol(W), 2, centre, "+")
}

# logistic elastic net by coordinate descent on standardised features, the objective of glmnet:
#   -(1/n) loglik(b0, b) + lambda * sum((1 - alpha)/2 b_j^2 + alpha |b_j|).
# Same algorithm as the elastic-net demonstration (proximal Newton outer loop, cyclic coordinate
# descent inner loop, warm starts along the path). Returns coefficients on the original scale.
soft <- function(z, g) sign(z) * pmax(abs(z) - g, 0)
enet_logistic <- function(x, y01, lambda, alpha = 1, tol = 1e-12, maxit = 10000) {
  n <- nrow(x); p <- ncol(x)
  xm <- colMeans(x); xs <- sqrt(colMeans(sweep(x, 2, xm)^2))
  Z <- sweep(sweep(x, 2, xm), 2, xs, "/")
  b0 <- log(mean(y01) / (1 - mean(y01))); b <- numeric(p)
  out <- matrix(0, p + 1, length(lambda))
  for (l in seq_along(lambda)) {
    lam <- lambda[l]
    for (outer in seq_len(200)) {
      eta <- b0 + drop(Z %*% b)
      pr <- 1 / (1 + exp(-eta)); w <- pmax(pr * (1 - pr), 1e-10)
      zres <- (y01 - pr) / w        # working residual: z - eta
      r <- zres
      bold <- c(b0, b)
      v <- colSums(w * Z^2) / n
      for (it in seq_len(maxit)) {
        dmax <- 0
        d0 <- sum(w * r) / sum(w); b0 <- b0 + d0; r <- r - d0
        dmax <- max(dmax, sum(w) / n * d0^2)
        for (j in seq_len(p)) {
          bj <- b[j]
          g <- sum(w * Z[, j] * r) / n + v[j] * bj
          nb <- soft(g, lam * alpha) / (v[j] + lam * (1 - alpha))
          if (nb != bj) { r <- r - (nb - bj) * Z[, j]; b[j] <- nb; dmax <- max(dmax, v[j] * (nb - bj)^2) }
        }
        if (dmax < tol) break
      }
      if (max(abs(c(b0, b) - bold)) < 1e-9) break
    }
    out[, l] <- c(b0 - sum(b * xm / xs), b / xs)
  }
  out
}

# area under the ROC curve as the Mann-Whitney probability P(score+ > score-) + ties/2
auc_mw <- function(score, y01) {
  pos <- score[y01 == 1]; neg <- score[y01 == 0]
  mean(outer(pos, neg, ">") + 0.5 * outer(pos, neg, "=="))
}

# =============================================================================================
# Figure 12.1: two loess fits with different spans

fig12_1_data <- function() {
  set.seed(121)
  x <- 0:30
  data.frame(x = x, y = 2 + 0.01 * x^2 + 0.1 * x + 2 * rnorm(length(x)))
}
fig_12_1 <- function() {
  ov <- fig12_1_data()
  p <- ggplot(ov, aes(x, y)) + geom_point(size = 1.4) +
    geom_smooth(method = "loess", formula = y ~ x, span = 0.2, colour = "dodgerblue3", se = FALSE, linewidth = 0.8) +
    geom_smooth(method = "loess", formula = y ~ x, span = 0.8, colour = "darkorange1", se = FALSE, linewidth = 0.8)
  savefig(p, "fig-12-1", ch = CH, width = 3.6, height = 3.2)
}

# =============================================================================================
# Section 12.2 and 12.3.1: the diabetes data

fig_12_3 <- function() {
  db <- load_diabetes()
  long <- pivot_longer(db, -group, names_to = "variable", values_to = "value")
  long$variable <- factor(long$variable, levels = c("rw", "fpg", "glucose", "insulin", "sspg"))
  p <- ggplot(long, aes(value, colour = group)) + geom_density(linewidth = 0.6) +
    facet_wrap(~variable, ncol = 1, scales = "free") +
    scale_colour_manual(values = GROUP_COL) + theme(legend.position = "bottom")
  savefig(p, "fig-12-3", ch = CH, width = 3.8, height = 7.5)
}

diabetes_scatter <- function(db) {
  ggplot(mapping = aes(sspg, glucose)) + geom_point(aes(colour = group), data = db, size = 1.3) +
    scale_colour_manual(values = GROUP_COL) + scale_fill_manual(values = GROUP_COL)
}
fig_12_5 <- function() {
  p <- diabetes_scatter(load_diabetes())
  savefig(p, "fig-12-5", ch = CH, width = 4.6, height = 3.6)
}

lda_region_plot <- function(db, prior = NULL) {
  fit <- if (is.null(prior)) lda(group ~ sspg + glucose, data = db) else lda(group ~ sspg + glucose, data = db, prior = prior)
  grid <- expand.grid(sspg = make1Dgrid(db$sspg), glucose = make1Dgrid(db$glucose))
  grid$ghat <- predict(fit, newdata = grid)$class
  W <- pooled_cov(db[, c("sspg", "glucose")], db$group)
  centres <- as.data.frame(fit$means)
  ell <- do.call(rbind, lapply(rownames(fit$means), function(gr) {
    e <- ellipse_pts(fit$means[gr, ], W)
    data.frame(sspg = e[, 1], glucose = e[, 2], group = gr)
  }))
  ell$group <- factor(ell$group, levels = levels(db$group))
  diabetes_scatter(db) +
    geom_raster(aes(fill = ghat), data = grid, alpha = 0.25, interpolate = TRUE) +
    geom_point(data = centres, shape = 3, size = 5, stroke = 1.2) +
    geom_path(aes(colour = group), data = ell, linewidth = 0.7) +
    scale_x_continuous(expand = c(0, 0)) + scale_y_continuous(expand = c(0, 0)) +
    labs(fill = "predicted", colour = "group")
}
fig_12_6 <- function() {
  savefig(lda_region_plot(load_diabetes()), "fig-12-6", ch = CH, width = 5.4, height = 4)
}
fig_12_7 <- function() {
  db <- load_diabetes()
  savefig(lda_region_plot(db, prior = rep(1 / 3, 3)), "fig-12-7", ch = CH, width = 5.4, height = 4)
}

# =============================================================================================
# Section 12.3.2: embryonic cell states

EMB_COL <- c(E3.25 = unname(COL["blue"]), E3.5 = unname(COL["orange"]), E4.5 = unname(COL["green"]))
fig_12_8 <- function() {
  ec <- load_embryo()
  genes <- c("Fn1", "Timd2", "Gata4", "Sox7")
  p <- GGally::ggpairs(ec, mapping = aes(colour = Embryonic.day), columns = genes,
                       upper = list(continuous = GGally::wrap("points", size = 0.8)),
                       lower = list(continuous = GGally::wrap("points", size = 0.8)),
                       diag = list(continuous = GGally::wrap("densityDiag", alpha = 0.5))) +
    scale_colour_manual(values = EMB_COL) + scale_fill_manual(values = EMB_COL) + theme_course(9)
  savefig(p, "fig-12-8", ch = CH, width = 6.4, height = 6.4)
}

fig_12_9 <- function() {
  ec <- load_embryo()
  fit <- lda(Embryonic.day ~ Fn1 + Timd2 + Gata4 + Sox7, data = ec)
  pr <- predict(fit)
  rot <- data.frame(pr$x, ed = ec$Embryonic.day)
  # exact LDA regions in the discriminant plane: with K = 3 classes the two discriminant coordinates
  # carry all the information, and the rule is "nearest projected centre, corrected by log prior"
  m <- colSums(fit$prior * fit$means)
  dm <- sweep(fit$means, 2, m) %*% fit$scaling
  grid <- expand.grid(LD1 = make1Dgrid(rot$LD1), LD2 = make1Dgrid(rot$LD2))
  G <- as.matrix(grid)
  dist <- sapply(seq_len(nrow(dm)), function(k) 0.5 * rowSums(sweep(G, 2, dm[k, ])^2) - log(fit$prior[k]))
  grid$edhat <- factor(levels(ec$Embryonic.day)[max.col(-dist)], levels = levels(ec$Embryonic.day))
  p <- ggplot() + geom_raster(aes(LD1, LD2, fill = edhat), data = grid, alpha = 0.35, interpolate = TRUE) +
    geom_point(aes(LD1, LD2, colour = ed), data = rot, size = 1.4) +
    scale_colour_manual(values = EMB_COL) + scale_fill_manual(values = EMB_COL) +
    scale_x_continuous(expand = c(0, 0)) + scale_y_continuous(expand = c(0, 0)) + coord_fixed() +
    labs(colour = "Embryonic.day", fill = "predicted")
  savefig(p, "fig-12-9", ch = CH, width = 5.6, height = 3.5)
}

fig_12_10 <- function() {
  ec <- load_embryo()
  fit <- qda(Embryonic.day ~ Fn1 + Timd2 + Gata4 + Sox7, data = ec)
  vars <- colnames(fit$means)
  prs <- combn(vars, 2)
  panels <- lapply(seq_len(ncol(prs)), function(i) {
    a <- prs[1, i]; b <- prs[2, i]
    grid <- expand.grid(make1Dgrid(ec[[a]]), make1Dgrid(ec[[b]]))
    colnames(grid) <- c(a, b)
    for (v in setdiff(vars, c(a, b))) grid[[v]] <- median(ec[[v]])
    grid$edhat <- predict(fit, newdata = grid)$class
    ggplot() + geom_raster(aes(.data[[a]], .data[[b]], fill = edhat), data = grid, alpha = 0.35, interpolate = TRUE) +
      geom_point(aes(.data[[a]], .data[[b]], colour = Embryonic.day), data = ec, size = 1) +
      scale_colour_manual(values = EMB_COL) + scale_fill_manual(values = EMB_COL) +
      scale_x_continuous(expand = c(0, 0)) + scale_y_continuous(expand = c(0, 0)) + coord_fixed() +
      theme_course(9) + theme(legend.position = if (i == ncol(prs)) "right" else "none") +
      labs(fill = "predicted")
  })
  g <- gridExtra::arrangeGrob(grobs = panels, ncol = 2)
  savefig(g, "fig-12-10", ch = CH, width = 7.6, height = 9)
}

# =============================================================================================
# Section 12.4: learning by heart, cross-validation, the curse of dimensionality

fig_12_11 <- function() {
  res <- cached("fig12-11", {
    set.seed(1211)
    n <- 20
    do.call(rbind, lapply(2:21, function(pp) {
      m <- replicate(100, {
        xmat <- matrix(rnorm(n * pp), nrow = n)
        resp <- sample(c("apple", "orange"), n, replace = TRUE)
        fit <- suppressWarnings(lda(xmat, resp))
        mean(predict(fit)$class != resp)
      })
      data.frame(p = pp, mcl = mean(m))
    }))
  })
  p <- ggplot(res, aes(p, mcl)) + geom_line() + geom_point(size = 1.4) + ylab("Misclassification rate")
  savefig(p, "fig-12-11", ch = CH, width = 3.6, height = 3.2)
}

fig12_12_data <- function() {
  set.seed(1212)
  n <- 20; pmax <- 21
  xmat <- round(matrix(rnorm(n * pmax), nrow = n), 3)
  resp <- sample(c("apple", "orange"), n, replace = TRUE)
  list(x = xmat, resp = resp)
}
fig12_12_curve <- function(d = fig12_12_data()) {
  do.call(rbind, lapply(2:21, function(k) { r <- loo_lda(d$x[, 1:k, drop = FALSE], d$resp); data.frame(p = k, train = r[["train"]], test = r[["test"]]) }))
}
fig_12_12 <- function() {
  res <- cached("fig12-12", fig12_12_curve())
  long <- pivot_longer(res, -p, names_to = "variable", values_to = "value")
  p <- ggplot(long, aes(p, value, colour = variable)) + geom_line() + geom_point(size = 1.4) +
    scale_colour_manual(values = c(train = unname(COL["blue"]), test = unname(COL["red"]))) +
    ylab("Misclassification rate")
  savefig(p, "fig-12-12", ch = CH, width = 4.2, height = 3.2)
}

# n = 20, 20 features, the first six shifted by 1 (apples) or 2 (oranges)
sim12_13 <- function(n = 20, pmax = 20) {
  xmat <- matrix(rnorm(n * pmax), nrow = n)
  resp <- sample(c("apple", "orange"), n, replace = TRUE)
  xmat[, 1:6] <- xmat[, 1:6] + as.integer(factor(resp))
  list(x = round(xmat, 3), resp = resp)
}
fig12_13_res <- function() {
  cached("fig12-13", {
    set.seed(1213)
    reps <- lapply(1:100, function(r) {
      d <- sim12_13()
      cur <- do.call(rbind, lapply(2:20, function(k) { e <- loo_lda(d$x[, 1:k, drop = FALSE], d$resp); data.frame(p = k, train = e[["train"]], test = e[["test"]]) }))
      list(data = d, curve = cur)
    })
    curves <- do.call(rbind, lapply(seq_along(reps), function(r) cbind(rep = r, reps[[r]]$curve)))
    list(first = reps[[1]], curves = curves)
  })
}
fig_12_13 <- function() {
  res <- fig12_13_res()
  avg <- res$curves %>% group_by(p) %>% summarise(train = mean(train), test = mean(test), .groups = "drop")
  long <- pivot_longer(avg, -p, names_to = "variable", values_to = "value")
  p <- ggplot(long, aes(p, value, colour = variable)) + geom_line() + geom_point(size = 1.4) +
    scale_colour_manual(values = c(train = unname(COL["blue"]), test = unname(COL["red"]))) +
    ylab("Misclassification rate")
  savefig(p, "fig-12-13", ch = CH, width = 4.2, height = 3.2)
}

fig_12_15 <- function() {
  side <- function(p, N = 1e6, k = 10) (k / N)^(1 / p)
  d <- data.frame(p = 1:400, side = side(1:400))
  p <- ggplot(d, aes(p, side)) + geom_line(colour = "red") + geom_hline(yintercept = 1, linetype = 2) +
    ylab("side length")
  savefig(p, "fig-12-15", ch = CH, width = 3.8, height = 2.7)
}

fig_12_16 <- function() {
  d <- data.frame(p = 1:400)
  d$shell <- 1 - 0.98^d$p
  p <- ggplot(d, aes(p, shell)) + geom_line(colour = "blue") + ylab("V(shell)")
  savefig(p, "fig-12-16", ch = CH, width = 3.8, height = 2.7)
}

fig12_17_data <- function() {
  set.seed(1217)
  n <- 1000
  p <- round(10^seq(0, 4, by = 0.25))
  cv <- vapply(p, function(k) {
    x1 <- matrix(runif(k * n), nrow = n); x2 <- matrix(runif(k * n), nrow = n)
    d <- sqrt(rowSums((x1 - x2)^2))
    sd(d) / mean(d)
  }, numeric(1))
  data.frame(p = p, cv = cv)
}
fig_12_17 <- function() {
  d <- cached("fig12-17", fig12_17_data())
  # the large-p approximation sqrt(7/(20 p)) derived in the notes
  d2 <- data.frame(lp = seq(0, 4, length.out = 200)); d2$approx <- sqrt(7 / (20 * 10^d2$lp))
  p <- ggplot(d, aes(log10(p), cv)) + geom_line(colour = "orange") + geom_point(size = 1.3) +
    geom_line(aes(lp, approx), data = d2, linetype = 2, colour = "grey40") + ylab("CV of distance")
  savefig(p, "fig-12-17", ch = CH, width = 3.8, height = 2.7)
}

# =============================================================================================
# Section 12.6.2: colon cancer and the stool microbiome

fig_12_19 <- function() {
  z <- load_zeller()
  sel <- c(510, 527)
  d <- do.call(rbind, lapply(sel, function(j) data.frame(feature = sprintf("%d: %s", j, short_taxon(colnames(z$X)[j])), value = z$X[, j])))
  p <- ggplot(d, aes(value)) + geom_histogram(bins = 25, fill = "grey35") +
    facet_wrap(~feature, ncol = 1, scales = "free")
  savefig(p, "fig-12-19", ch = CH, width = 3.6, height = 4.2)
}

zeller_glmnet <- function() cached("zeller-glmnet", {
  z <- load_zeller()
  glmnet(z$X, z$disease, family = "binomial")
})
zeller_cv <- function(transform = c("raw", "asinh")) {
  transform <- match.arg(transform)
  cached(paste0("zeller-cv-", transform), {
    z <- load_zeller()
    X <- if (transform == "asinh") asinh(z$X) else z$X
    set.seed(1221)
    foldid <- sample(rep(1:10, length.out = nrow(X)))
    cv.glmnet(X, z$disease, family = "binomial", foldid = foldid)
  })
}

fig_12_20 <- function() {
  fit <- zeller_glmnet()
  B <- as.matrix(fit$beta)
  l1 <- colSums(abs(B))
  nz <- rowSums(B != 0) > 0
  d <- data.frame(feature = rep(rownames(B)[nz], times = ncol(B)), l1 = rep(l1, each = sum(nz)),
                  coef = as.vector(B[nz, ]))
  d$col <- factor(as.integer(factor(d$feature)) %% 8)
  df <- fit$df
  pv <- pretty(range(l1), 5); pv <- pv[pv <= max(l1)]
  ticks <- unique(vapply(pv, function(v) which.min(abs(l1 - v)), 1L))
  p <- ggplot(d, aes(l1, coef, group = feature, colour = col)) + geom_line(linewidth = 0.45) +
    scale_colour_brewer(palette = "Dark2", guide = "none") +
    annotate("text", x = l1[ticks], y = Inf, label = df[ticks], vjust = 1.4, size = 3, colour = "grey30") +
    labs(x = "L1 norm of the coefficients", y = "Coefficients")
  savefig(p, "fig-12-20", ch = CH, width = 4.4, height = 3.8)
}

cv_plot <- function(cvfit, ylab) {
  d <- data.frame(ll = log(cvfit$lambda), cvm = cvfit$cvm, lo = cvfit$cvlo, hi = cvfit$cvup, nz = cvfit$nzero)
  ticks <- unique(round(seq(1, nrow(d), length.out = 8)))
  ggplot(d, aes(ll, cvm)) + geom_errorbar(aes(ymin = lo, ymax = hi), colour = "grey60", width = 0.04) +
    geom_point(colour = "red", size = 1.2) +
    geom_vline(xintercept = log(c(cvfit$lambda.min, cvfit$lambda.1se)), linetype = 2) +
    annotate("text", x = d$ll[ticks], y = Inf, label = d$nz[ticks], vjust = 1.4, size = 3, colour = "grey30") +
    labs(x = expression(log(lambda)), y = ylab)
}
fig_12_21 <- function() {
  savefig(cv_plot(zeller_cv("raw"), "Binomial deviance"), "fig-12-21", ch = CH, width = 4.4, height = 3.8)
}
fig_12_22 <- function() {
  savefig(cv_plot(zeller_cv("asinh"), "Binomial deviance"), "fig-12-22", ch = CH, width = 4.4, height = 3.8)
}

# =============================================================================================
# Section 12.6.3: genotype of the E3.25 mouse cells

mouse_cv <- function() cached("mouse-cv", {
  x <- load_hiiragi()
  sx <- x[, x$Embryonic.day == "E3.25"]
  set.seed(1223)
  foldid <- sample(rep(1:10, length.out = ncol(sx)))
  f <- cv.glmnet(t(Biobase::exprs(sx)), factor(sx$genotype), family = "binomial",
                 type.measure = "class", foldid = foldid)
  f$glmnet.fit <- NULL   # keep the cache small
  f
})
fig_12_23 <- function() {
  savefig(cv_plot(mouse_cv(), "Misclassification Error"), "fig-12-23", ch = CH, width = 4.4, height = 3.8)
}

mouse_ttests <- function() cached("mouse-tt", {
  x <- load_hiiragi()
  sx <- x[, x$Embryonic.day == "E3.25"]
  genefilter::rowttests(sx, "genotype")
})
fig_12_24 <- function() {
  tt <- mouse_ttests()
  p <- ggplot(tt, aes(p.value)) + geom_histogram(breaks = seq(0, 1, by = 0.01), fill = "grey35") +
    labs(x = "p-value", y = "count")
  savefig(p, "fig-12-24", ch = CH, width = 4.4, height = 2.8)
}

# =============================================================================================
# Section 12.7: caret

nnet_tuning <- function() cached("nnet", {
  suppressPackageStartupMessages(library(caret))
  ec <- load_embryo()
  set.seed(1225)
  ctrl <- trainControl(method = "repeatedcv", number = 10, repeats = 3, classProbs = TRUE)
  grid <- expand.grid(size = c(2, 4, 8), decay = c(0, 1e-2, 1e-1))
  fit <- train(Embryonic.day ~ Fn1 + Timd2 + Gata4 + Sox7, data = ec, method = "nnet",
               tuneGrid = grid, trControl = ctrl, metric = "Accuracy", trace = FALSE)
  list(results = fit$results, bestTune = fit$bestTune, n_methods = length(caret::getModelInfo()),
       pred = as.character(predict(fit)), truth = as.character(ec$Embryonic.day))
})
fig_12_25 <- function() {
  r <- nnet_tuning()$results
  r$decay <- factor(r$decay)
  p <- ggplot(r, aes(size, Accuracy, colour = decay, shape = decay)) + geom_line() + geom_point(size = 2) +
    scale_colour_manual(values = GROUP_COL) + scale_x_continuous(breaks = c(2, 4, 8)) +
    labs(x = "#Hidden Units", y = "Accuracy (Repeated Cross-Validation)", colour = "Weight Decay", shape = "Weight Decay") +
    theme(legend.position = "top")
  savefig(p, "fig-12-25", ch = CH, width = 4.2, height = 4.4)
}

# =============================================================================================
# numbers quoted in the notes (questions and exercises)

results_ch12 <- function() {
  res <- list()
  db <- load_diabetes()
  f2 <- lda(group ~ sspg + glucose, data = db)
  g2 <- predict(f2)$class
  res$lda2_table <- table(ghat = g2, truth = db$group)
  res$lda2_err <- mean(g2 != db$group)
  res$lda2_scaling <- f2$scaling
  res$lda2_trace <- f2$svd^2 / sum(f2$svd^2)
  fu <- lda(group ~ sspg + glucose, data = db, prior = rep(1 / 3, 3))
  res$lda2_uniform_err <- mean(predict(fu)$class != db$group)
  res$lda2_uniform_table <- table(ghat = predict(fu)$class, truth = db$group)
  f5 <- lda(group ~ rw + fpg + glucose + sspg + insulin, data = db)
  res$lda5_err <- mean(predict(f5)$class != db$group)
  res$lda5_table <- table(ghat = predict(f5)$class, truth = db$group)
  res$lda2_loo <- mean(lda(group ~ sspg + glucose, data = db, CV = TRUE)$class != db$group)
  res$lda5_loo <- mean(lda(group ~ rw + fpg + glucose + sspg + insulin, data = db, CV = TRUE)$class != db$group)
  res$qda2_err <- mean(predict(qda(group ~ sspg + glucose, data = db))$class != db$group)
  res$qda2_loo <- mean(qda(group ~ sspg + glucose, data = db, CV = TRUE)$class != db$group)
  res$qda5_loo <- mean(qda(group ~ rw + fpg + glucose + sspg + insulin, data = db, CV = TRUE)$class != db$group)

  ec <- load_embryo()
  fe <- lda(Embryonic.day ~ Fn1 + Timd2 + Gata4 + Sox7, data = ec)
  res$embryo_scaling <- fe$scaling
  res$embryo_lda_err <- mean(predict(fe)$class != ec$Embryonic.day)
  res$embryo_qda_err <- mean(predict(qda(Embryonic.day ~ Fn1 + Timd2 + Gata4 + Sox7, data = ec))$class != ec$Embryonic.day)
  res$embryo_n <- table(ec$Embryonic.day)

  x <- load_hiiragi()
  X1000 <- t(Biobase::exprs(x))[, 1:1000]
  res$lda1000_warning <- tryCatch({ withCallingHandlers(lda(X1000, x$Embryonic.day), warning = function(w) stop(conditionMessage(w))); "none" },
                                  error = function(e) conditionMessage(e))
  res$qda1000_error <- tryCatch({ qda(X1000, x$Embryonic.day); "none" }, error = function(e) conditionMessage(e))

  # microbiome: lasso at s = 0.04 and at lambda.1se
  z <- load_zeller(); gf <- zeller_glmnet(); cvr <- zeller_cv("raw"); cva <- zeller_cv("asinh")
  res$zeller_n <- table(z$disease)
  res$zeller_s004 <- table(pred = predict(gf, newx = z$X, type = "class", s = 0.04)[, 1], truth = z$disease)
  res$zeller_lambda <- c(min = cvr$lambda.min, se1 = cvr$lambda.1se)
  res$zeller_asinh_lambda <- c(min = cva$lambda.min, se1 = cva$lambda.1se)
  res$zeller_cvmin <- c(raw = min(cvr$cvm), asinh = min(cva$cvm))
  res$zeller_1se_table <- table(pred = predict(gf, newx = z$X, type = "class", s = cvr$lambda.1se)[, 1], truth = z$disease)
  cf <- coef(gf, s = cvr$lambda.1se)[-1, 1]
  top <- order(abs(cf), decreasing = TRUE)[1:3]
  res$zeller_top <- data.frame(taxon = vapply(names(cf)[top], short_taxon, ""), coef = unname(cf[top]))
  res$zeller_nz_1se <- sum(cf != 0)

  mc <- mouse_cv()
  res$mouse_cv <- c(lambda.min = mc$lambda.min, cvm_first = mc$cvm[1], cvm_min = min(mc$cvm), cvm_last = mc$cvm[length(mc$cvm)])
  tt <- mouse_ttests()
  res$mouse_frac_p001 <- mean(tt$p.value < 0.01)
  res$mouse_pi0 <- min(1, mean(tt$p.value > 0.5) / 0.5)
  # one nearest neighbour on all scaled features
  d <- as.matrix(dist(scale(t(Biobase::exprs(x)))))
  diag(d) <- Inf
  nn <- apply(d, 2, which.min)
  res$nn_table <- table(x$sampleGroup, x$sampleGroup[nn])
  res$nn_err <- mean(x$sampleGroup != x$sampleGroup[nn])

  nt <- nnet_tuning()
  res$nnet <- nt$results[, c("size", "decay", "Accuracy", "Kappa")]
  res$nnet_best <- nt$bestTune
  res$caret_methods <- nt$n_methods

  # feature selection inside and outside the cross-validation loop (50 data sets of pure noise)
  res$selection <- cached("selection", {
    set.seed(1240)
    t(replicate(50, {
      xx <- matrix(rnorm(40 * 500), 40)
      rr <- rep(c("a", "b"), each = 20)
      selection_cv(xx, rr, 10)
    }))
  })

  # Exercise 12.1: kernel SVMs on the microbiome data, 10-fold cross-validated error
  res$svm <- cached("svm", {
    suppressPackageStartupMessages(library(kernlab))
    out <- list()
    for (tr in c("raw", "asinh")) {
      X <- if (tr == "asinh") asinh(z$X) else z$X
      X <- X[, apply(X, 2, sd) > 0]
      for (k in c("vanilladot", "rbfdot", "polydot", "laplacedot")) {
        errs <- vapply(1:5, function(r) {
          set.seed(1250 + r)
          kp <- switch(k, vanilladot = list(), rbfdot = list(sigma = 1 / ncol(X)), polydot = list(degree = 2),
                       laplacedot = list(sigma = 1 / ncol(X)))
          m <- ksvm(X, z$disease, type = "C-svc", kernel = k, kpar = kp, C = 1, cross = 10, scaled = TRUE)
          cross(m)
        }, numeric(1))
        out[[length(out) + 1]] <- data.frame(transform = tr, kernel = k, cv_error = mean(errs), sd = sd(errs))
      }
    }
    do.call(rbind, out)
  })

  # Exercise 12.2: prostate cancer data (Stamey et al. 1989), ridge versus lasso
  res$prostate <- prostate_ex()

  # Exercise 12.3: choosing the loess span by leave-one-out cross-validation
  ov <- fig12_1_data()
  spans <- seq(0.15, 1, by = 0.05)
  cvs <- vapply(spans, function(s) mean(vapply(seq_len(nrow(ov)), function(i) {
    f <- loess(y ~ x, data = ov[-i, ], span = s, control = loess.control(surface = "direct"))
    (ov$y[i] - predict(f, newdata = ov[i, ]))^2 }, numeric(1))), numeric(1))
  res$span_cv <- data.frame(span = spans, cv = cvs)
  res
}

prostate_ex <- function() {
  f <- file.path(CACHE, "ch12-prostate.data")
  if (!file.exists(f)) {
    ok <- tryCatch({ utils::download.file("https://hastie.su.domains/ElemStatLearn/datasets/prostate.data", f, quiet = TRUE); TRUE },
                   error = function(e) FALSE)
    if (!ok) return(NULL)
  }
  pr <- read.table(f, header = TRUE)
  vars <- c("lcavol", "lweight", "age", "lbph", "svi", "lcp", "gleason", "pgg45")
  tr <- as.character(pr$train) %in% c("T", "TRUE")
  X <- as.matrix(pr[, vars]); y <- pr$lpsa
  set.seed(1260)
  foldid <- sample(rep(1:10, length.out = sum(tr)))
  ols <- lm(y ~ X, subset = tr)
  te_ols <- mean((y[!tr] - cbind(1, X[!tr, ]) %*% coef(ols))^2)
  out <- data.frame(method = "least squares", lambda = 0, nonzero = length(vars), test_mse = te_ols)
  coefs <- list(ols = coef(ols)[-1])
  for (a in c(0, 1)) {
    cvf <- cv.glmnet(X[tr, ], y[tr], alpha = a, foldid = foldid)
    pr1 <- predict(cvf, newx = X[!tr, ], s = "lambda.1se")
    b <- coef(cvf, s = "lambda.1se")[-1, 1]
    out <- rbind(out, data.frame(method = if (a == 0) "ridge" else "lasso", lambda = cvf$lambda.1se,
                                 nonzero = sum(b != 0), test_mse = mean((y[!tr] - pr1)^2)))
    coefs[[if (a == 0) "ridge" else "lasso"]] <- b
  }
  list(table = out, coefs = do.call(cbind, coefs), n_train = sum(tr), n_test = sum(!tr))
}

# =============================================================================================
# JSON for the browser demonstrations

data_ch12 <- function() {
  db <- load_diabetes()
  export_json(list(source = "rrcov::diabetes (Reaven and Miller 1979)",
                   vars = c("rw", "fpg", "glucose", "insulin", "sspg"),
                   groups = levels(db$group),
                   rw = db$rw, fpg = db$fpg, glucose = db$glucose, insulin = db$insulin, sspg = db$sspg,
                   group = as.integer(db$group) - 1L), "ch12-diabetes.json")

  d12 <- fig12_12_data()
  c12 <- cached("fig12-12", fig12_12_curve(d12))
  r13 <- fig12_13_res()
  c13 <- subset(r13$curves, rep == 1)
  set.seed(1241)
  xs <- round(matrix(rnorm(40 * 500), 40), 3)
  rs <- rep(c("a", "b"), each = 20)
  sel <- vapply(c(1, 2, 5, 10, 20, 50), function(k) selection_cv(xs, rs, k), numeric(2))
  pack <- function(x) lapply(seq_len(nrow(x)), function(i) x[i, ])
  export_json(list(
    noise = list(note = "Figure 12.12 data: n = 20, 21 standard normal features, random labels",
                 x = pack(d12$x), y = as.integer(d12$resp == "orange"),
                 p = c12$p, train = c12$train, test = c12$test),
    signal = list(note = "first replicate of Figure 12.13: features 1-6 shifted by the label (1 or 2)",
                  x = pack(r13$first$data$x), y = as.integer(r13$first$data$resp == "orange"),
                  p = c13$p, train = c13$train, test = c13$test),
    selection = list(note = "n = 40 (20 per class), 500 standard normal features, top-k by |t| then LDA",
                     x = pack(xs), y = as.integer(rs == "b"), k = c(1, 2, 5, 10, 20, 50),
                     outside = sel["outside", ], inside = sel["inside", ])
  ), "ch12-cv.json", digits = 5)

  zs <- zeller_species()
  set.seed(1242)
  foldid <- sample(rep(1:10, length.out = nrow(zs$X)))
  export_json(list(source = "Zeller et al. (2014), ExperimentHub EH361: normal (n) and cancer samples",
                   note = "species-level relative abundances (percent), present in >= 10% of samples, 4 significant digits",
                   taxa = zs$short, y = as.integer(zs$y == "n"), labels = levels(zs$y),
                   foldid = foldid, x = pack(zs$X)), "ch12-zeller.json", digits = 4)
}

results_out <- function() {
  res <- results_ch12()
  saveRDS(res, cache_path("results"))
  str(res, max.level = 1, give.attr = FALSE)
  print(res[c("lda2_table", "lda2_err", "lda2_uniform_err", "lda5_err", "lda2_loo", "lda5_loo", "qda2_err", "qda2_loo",
              "qda5_loo", "embryo_lda_err", "embryo_qda_err", "lda1000_warning", "qda1000_error",
              "zeller_s004", "zeller_lambda", "zeller_asinh_lambda", "zeller_cvmin", "zeller_1se_table",
              "zeller_top", "zeller_nz_1se", "mouse_cv", "mouse_frac_p001", "mouse_pi0", "nn_err", "nnet_best",
              "caret_methods", "svm", "prostate", "span_cv")])
  cat("selection (mean over 50 data sets):", colMeans(res$selection), "\n")
}

# Set .ch12_no_run before sourcing this file to reuse the helpers without plotting
if (!exists(".ch12_no_run")) {
  run_figures(list(fig_12_1 = fig_12_1, fig_12_3 = fig_12_3, fig_12_5 = fig_12_5, fig_12_6 = fig_12_6,
                   fig_12_7 = fig_12_7, fig_12_8 = fig_12_8, fig_12_9 = fig_12_9, fig_12_10 = fig_12_10,
                   fig_12_11 = fig_12_11, fig_12_12 = fig_12_12, fig_12_13 = fig_12_13, fig_12_15 = fig_12_15,
                   fig_12_16 = fig_12_16, fig_12_17 = fig_12_17, fig_12_19 = fig_12_19, fig_12_20 = fig_12_20,
                   fig_12_21 = fig_12_21, fig_12_22 = fig_12_22, fig_12_23 = fig_12_23, fig_12_24 = fig_12_24,
                   fig_12_25 = fig_12_25, data_ch12 = data_ch12, results = results_out))
}
