# Chapter 13 lab: multiple testing (R version).
# Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie and
# Tibshirani (2021), An Introduction to Statistical Learning with Applications in R, 2nd edition,
# Section 13.6, and the Python lab of James et al. (2023), Section 13.6.
#
# The lab works through
#   13.6.1  one-sample t-tests on simulated data, and the 2 x 2 table of Table 13.2;
#   13.6.2  the family-wise error rate: the curves (13.5), t-tests for five fund managers,
#           Bonferroni and Holm (p.adjust, and by hand), a paired t-test, Tukey's method (TukeyHSD),
#           and, beyond the book's lab, Scheffe's threshold for the contrast (13.8);
#   13.6.3  the false discovery rate: t-tests for all 2,000 managers, Benjamini-Hochberg
#           (p.adjust, and by hand);
#   13.6.4  re-sampling on the Khan data: a permutation p-value (Algorithm 13.3) and the
#           plug-in FDR (Algorithm 13.4) for 100 random genes;
# and the two applied exercises (13.7.7 on Carseats, 13.7.8 on simulated fund managers).
# The random draws follow the R lab of the book (set.seed(6), set.seed(1)), so the printed numbers
# can be compared with it. The Python version is ch13_lab.py. Run from the code/ folder:
#   Rscript ch13_lab.R          (about 20 seconds)
# Figures go to ../img/ch13/lab-r-*.svg (and .pdf) and the key numbers to cache/ch13_lab_R.csv.

suppressPackageStartupMessages(library(ISLR2))   # Fund, Khan, Carseats
options(digits = 5, width = 100)

out_dir <- file.path("..", "img", "ch13")
dir.create(out_dir, showWarnings = FALSE, recursive = TRUE)
dir.create("cache", showWarnings = FALSE)
key <- list()                                 # key numbers, written to cache/ch13_lab_R.csv at the end
keep <- function(name, value) key[[name]] <<- unname(as.numeric(value))
save_plot <- function(name, width, height, draw) {   # the same plot as SVG (for the web) and PDF
  if (requireNamespace("svglite", quietly = TRUE)) svglite::svglite(file.path(out_dir, paste0(name, ".svg")), width, height)
  else grDevices::svg(file.path(out_dir, paste0(name, ".svg")), width, height)
  draw(); invisible(dev.off())
  pdf(file.path(out_dir, paste0(name, ".pdf")), width, height); draw(); invisible(dev.off())
}

# ================================================================== 13.6.1 review of hypothesis tests
set.seed(6)
x <- matrix(rnorm(10 * 100), 10, 100)        # 100 variables, 10 observations each
x[, 1:50] <- x[, 1:50] + 0.5                 # the first 50 have mean 0.5: H0 is false for them
tt1 <- t.test(x[, 1], mu = 0)
tt1                                          # t = 2.08, df = 9, p = 0.067: a Type II error
keep("sim1_t1", tt1$statistic); keep("sim1_p1", tt1$p.value)

p.values <- apply(x, 2, function(col) t.test(col, mu = 0)$p.value)
decision <- ifelse(p.values <= 0.05, "Reject H0", "Do not reject H0")
truth <- rep(c("H0 is False", "H0 is True"), each = 50)
tab1 <- table(decision, truth)
tab1                                         # W = 40, U = 47, S = 10, V = 3
keep("sim1_table", tab1)                     # column-major: W, S, U, V

x <- matrix(rnorm(10 * 100), 10, 100)        # a stronger signal: mean 1 for the false nulls
x[, 1:50] <- x[, 1:50] + 1
p.values <- apply(x, 2, function(col) t.test(col, mu = 0)$p.value)
decision <- ifelse(p.values <= 0.05, "Reject H0", "Do not reject H0")
tab2 <- table(decision, truth)
tab2                                         # W = 9, U = 49, S = 41, V = 1
keep("sim2_table", tab2)

# ================================================================== 13.6.2 the family-wise error rate
m <- 1:500
fwe <- sapply(c(0.05, 0.01, 0.001), function(a) 1 - (1 - a)^m)   # (13.5), one column per alpha
save_plot("lab-r-fwer", 6, 4.5, function() {
  matplot(m, fwe, type = "l", lty = 1, lwd = 2, log = "x", ylim = c(0, 1), col = c(2, 4, 3),
          xlab = "Number of Hypotheses", ylab = "Family-Wise Error Rate")
  abline(h = 0.05, lty = 2)
  legend("topleft", legend = c("alpha = 0.05", "alpha = 0.01", "alpha = 0.001"), col = c(2, 4, 3), lwd = 2, bty = "n")
})
keep("fwer_100_005", fwe[100, 1])            # 0.994

fund.mini <- Fund[, 1:5]
t.test(fund.mini[, 1], mu = 0)               # t = 2.86, p = 0.0062
fund.pvalue <- sapply(fund.mini, function(col) t.test(col, mu = 0)$p.value)
fund.pvalue                                  # 0.0062 0.9183 0.0116 0.6005 0.7558
keep("mini_t", sapply(fund.mini, function(col) t.test(col, mu = 0)$statistic))
keep("mini_p", fund.pvalue)
keep("mini_mean", colMeans(fund.mini)); keep("mini_sd", apply(fund.mini, 2, sd))

p_bonf <- p.adjust(fund.pvalue, method = "bonferroni")
p_bonf                                       # only Manager One is below 0.05
all.equal(p_bonf, pmin(fund.pvalue * 5, 1))  # Bonferroni: multiply by m, cap at 1
p_holm <- p.adjust(fund.pvalue, method = "holm")
p_holm                                       # Managers One and Three are below 0.05
keep("mini_bonf", p_bonf); keep("mini_holm", p_holm)

# Holm's adjusted p-values by hand: sort, multiply the j-th smallest by m + 1 - j,
# take the running maximum, cap at 1, and put back in the original order.
holm_by_hand <- function(p) {
  m <- length(p); o <- order(p)
  adj <- pmin(1, cummax((m + 1 - seq_len(m)) * p[o]))
  adj[order(o)]
}
all.equal(unname(holm_by_hand(fund.pvalue)), unname(p_holm))   # TRUE

colMeans(fund.mini)                          # 3.0 -0.1 2.8 0.5 0.3
tt_pair <- t.test(fund.mini[, 1], fund.mini[, 2], paired = TRUE)
tt_pair                                      # t = 2.13, p = 0.038
tt_two <- t.test(fund.mini[, 1], fund.mini[, 2], var.equal = TRUE)
tt_two$p.value                               # the unpaired two-sample test (13.1): 0.0349
keep("pair_p", tt_pair$p.value); keep("two_p", tt_two$p.value)

returns <- as.vector(as.matrix(fund.mini))   # one long vector, manager by manager
manager <- factor(rep(1:5, each = 50))
a1 <- aov(returns ~ manager)
tukey <- TukeyHSD(x = a1)
tukey                                        # 2-1: diff -3.1, p adj 0.1862
keep("tukey_diff", tukey$manager[, "diff"]); keep("tukey_lwr", tukey$manager[, "lwr"])
keep("tukey_padj", tukey$manager[, "p adj"])
save_plot("lab-r-tukey", 6, 5, function() { par(las = 1); plot(tukey) })

# Beyond the book's lab: the p-value threshold of Tukey's method for these ten comparisons,
# and Scheffe's test of the contrast (13.8), managers 1 and 3 against 2, 4 and 5.
df_res <- df.residual(a1)                    # 250 - 5 = 245
s2 <- sum(residuals(a1)^2) / df_res          # pooled within-manager variance
alpha_T <- 2 * pt(qtukey(0.95, 5, df_res) / sqrt(2), df_res, lower.tail = FALSE)
cvec <- c(1/2, -1/3, 1/2, -1/3, -1/3)
est <- sum(cvec * colMeans(fund.mini))
t_contrast <- est / sqrt(s2 * sum(cvec^2) / 50)
p_contrast <- 2 * pt(abs(t_contrast), df_res, lower.tail = FALSE)
crit_S <- sqrt(4 * qf(0.95, 4, df_res))      # Scheffe: reject if |t| > sqrt((G - 1) F)
alpha_S <- 2 * pt(crit_S, df_res, lower.tail = FALSE)
c(alpha_T = alpha_T, t = t_contrast, p = p_contrast, alpha_S = alpha_S)   # p = 0.004, alpha_S = 0.002
keep("alpha_T", alpha_T); keep("contrast_t", t_contrast); keep("contrast_p", p_contrast); keep("alpha_S", alpha_S)

# ================================================================== 13.6.3 the false discovery rate
fund.pvalues <- sapply(Fund, function(col) t.test(col, mu = 0)$p.value)   # 2,000 one-sample tests
q.values.BH <- p.adjust(fund.pvalues, method = "BH")
q.values.BH[1:10]                            # 0.0899 0.9915 0.1221 0.9234 0.9560 0.0751 ...
sum(q.values.BH <= 0.1)                      # 146 managers at an FDR of 10%
sum(fund.pvalues <= 0.1 / 2000)              # Bonferroni at 0.1 rejects none
keep("fund_p", fund.pvalues[c(1:10, 1000, 2000)]); keep("fund_q", q.values.BH[1:10])
keep("bh_counts", sapply(c(0.05, 0.1, 0.2, 0.3), function(q) sum(q.values.BH <= q)))
keep("bonf_counts", sapply(c(0.05, 0.1, 0.3), function(a) sum(fund.pvalues <= a / 2000)))
keep("holm_counts", sapply(c(0.05, 0.1, 0.3), function(a) sum(p.adjust(fund.pvalues, "holm") <= a)))
keep("p_below_001", sum(fund.pvalues < 0.001))

# Benjamini-Hochberg by hand (Algorithm 13.2): the largest j with p_(j) <= q j / m,
# then reject the j smallest p-values.
ps <- sort(fund.pvalues)
m <- length(fund.pvalues)
q <- 0.1
wh.ps <- which(ps <= q * (1:m) / m)
wh <- if (length(wh.ps) > 0) 1:max(wh.ps) else integer(0)
length(wh)                                   # 146 again
keep("bh_by_hand", length(wh))
save_plot("lab-r-bh", 6, 4.5, function() {
  plot(ps, log = "xy", ylim = c(4e-6, 1), xlab = "Index", ylab = "P-Value", pch = 20, cex = 0.5, col = "grey40")
  points(wh, ps[wh], col = 4, pch = 20, cex = 0.6)
  abline(a = 0, b = q / m, col = 2, untf = TRUE)  # untf = TRUE draws the straight line p = qj/m on log axes
  abline(h = 0.1 / 2000, col = 3)
})

# ================================================================== 13.6.4 a re-sampling approach
x <- rbind(Khan$xtrain, Khan$xtest)          # 83 patients, 2,308 genes
y <- c(as.numeric(Khan$ytrain), as.numeric(Khan$ytest))
dim(x); table(y)                             # classes 1-4: 11, 29, 18, 25 patients
x <- as.matrix(x)
x1 <- x[which(y == 2), ]
x2 <- x[which(y == 4), ]
n1 <- nrow(x1); n2 <- nrow(x2)
t.out <- t.test(x1[, 11], x2[, 11], var.equal = TRUE)
TT <- t.out$statistic
c(TT, p = t.out$p.value)                     # T = -2.0936, p = 0.0412
keep("g11_T", TT); keep("g11_p", t.out$p.value)

# The pooled two-sample t-statistic (13.11), the number t.test(var.equal = TRUE) reports,
# written out so that 10,000 permutations take a fraction of a second.
tstat <- function(a, b) {
  na <- length(a); nb <- length(b)
  s <- sqrt(((na - 1) * var(a) + (nb - 1) * var(b)) / (na + nb - 2))
  (mean(a) - mean(b)) / (s * sqrt(1 / na + 1 / nb))
}
all.equal(unname(tstat(x1[, 11], x2[, 11])), unname(TT))   # TRUE

set.seed(1)
B <- 10000
Tbs <- rep(NA, B)
for (b in 1:B) {                             # Algorithm 13.3: random relabellings of the 54 patients
  dat <- sample(c(x1[, 11], x2[, 11]))
  Tbs[b] <- tstat(dat[1:n1], dat[(n1 + 1):(n1 + n2)])
}
mean(abs(Tbs) >= abs(TT))                    # the re-sampling p-value, 0.0416
keep("g11_perm", mean(abs(Tbs) >= abs(TT)))
save_plot("lab-r-perm11", 6, 4.5, function() {
  hist(Tbs, breaks = 100, xlim = c(-4.2, 4.2), main = "", xlab = "Null Distribution of Test Statistic", col = 7)
  grid_t <- seq(-4.2, 4.2, len = 1000)
  lines(grid_t, dt(grid_t, df = n1 + n2 - 2) * 1000, col = 2, lwd = 3)   # bins of width 0.1: density x B x 0.1
  abline(v = TT, col = 4, lwd = 2)
  text(TT + 0.5, 350, paste("T = ", round(TT, 4), sep = ""), col = 4)
})

# Algorithm 13.4 on 100 randomly chosen genes, B = 10,000 relabellings each.
m <- 100
set.seed(1)
index <- sample(ncol(x1), m)
Ts <- rep(NA, m)
Ts.star <- matrix(NA, ncol = m, nrow = B)
for (j in 1:m) {
  k <- index[j]
  Ts[j] <- tstat(x1[, k], x2[, k])
  for (b in 1:B) {
    dat <- sample(c(x1[, k], x2[, k]))
    Ts.star[b, j] <- tstat(dat[1:n1], dat[(n1 + 1):(n1 + n2)])
  }
}
cs <- sort(abs(Ts))                          # the thresholds c: the observed |T| values
FDRs <- Rs <- Vs <- rep(NA, m)
for (j in 1:m) {
  R <- sum(abs(Ts) >= cs[j])
  V <- sum(abs(Ts.star) >= cs[j]) / B        # estimated expected number of false positives
  Rs[j] <- R
  Vs[j] <- V
  FDRs[j] <- V / R
}
max(Rs[FDRs <= 0.1])                         # 15 genes at an estimated FDR of 0.1
sort(index[abs(Ts) >= min(cs[FDRs < 0.1])])
max(Rs[FDRs <= 0.2])                         # 28 genes at 0.2
sort(index[abs(Ts) >= min(cs[FDRs < 0.2])])
keep("index", index); keep("R_fdr", c(max(Rs[FDRs <= 0.1]), max(Rs[FDRs <= 0.2])))
keep("genes_01", sort(index[abs(Ts) >= min(cs[FDRs < 0.1])]))
keep("fdr100", FDRs)
save_plot("lab-r-fdr100", 6, 4.5, function() {
  plot(Rs, FDRs, xlab = "Number of Rejections", type = "l", ylab = "False Discovery Rate", col = 4, lwd = 3)
})

# Beyond the book's lab: the gene of Figure 13.8, whose null distribution is far from t_52.
T877 <- tstat(x1[, 877], x2[, 877])
Tb877 <- replicate(B, { dat <- sample(c(x1[, 877], x2[, 877])); tstat(dat[1:n1], dat[(n1 + 1):(n1 + n2)]) })
c(T = T877, p_t = 2 * pt(-abs(T877), n1 + n2 - 2), p_perm = mean(abs(Tb877) >= abs(T877)))   # 0.571 against 0.67
keep("g877_T", T877); keep("g877_perm", mean(abs(Tb877) >= abs(T877)))

# ================================================================== exercises
# Exercise 7: Sales on each quantitative variable of Carseats, one simple regression at a time.
quant <- c("CompPrice", "Income", "Advertising", "Population", "Price", "Age", "Education")
ex7_p <- sapply(quant, function(v) summary(lm(Carseats$Sales ~ Carseats[[v]]))$coefficients[2, 4])
signif(ex7_p, 3)
names(ex7_p)[ex7_p <= 0.05]                  # (b) no correction
names(ex7_p)[p.adjust(ex7_p, "bonferroni") <= 0.05]   # (c) FWER 0.05 (Holm gives the same)
names(ex7_p)[p.adjust(ex7_p, "holm") <= 0.05]
names(ex7_p)[p.adjust(ex7_p, "BH") <= 0.2]    # (d) FDR 0.2
keep("ex7_p", ex7_p)

# Exercise 8: 100 simulated managers with no skill at all, 20 months each.
set.seed(1)
n <- 20
m <- 100
X <- matrix(rnorm(n * m), ncol = m)
ex8_p <- apply(X, 2, function(col) t.test(col, mu = 0)$p.value)
save_plot("lab-r-ex8", 6, 4.5, function() hist(ex8_p, breaks = 20, main = "", xlab = "p-value", col = "grey80"))
ex8 <- c(raw = sum(ex8_p <= 0.05), holm = sum(p.adjust(ex8_p, "holm") <= 0.05),
         bh = sum(p.adjust(ex8_p, "BH") <= 0.05))
ex8                                          # (b) a few false rejections; (c), (d) none
best <- order(colMeans(X), decreasing = TRUE)[1:10]   # (e) cherry-pick the 10 best managers
p_best <- ex8_p[best]
ex8_cherry <- c(holm = sum(p.adjust(p_best, "holm") <= 0.05), bh = sum(p.adjust(p_best, "BH") <= 0.05))
ex8_cherry
keep("ex8", ex8); keep("ex8_cherry", ex8_cherry); keep("ex8_pmin", min(ex8_p))

# ================================================================== key numbers
flat <- unlist(lapply(names(key), function(k) {
  v <- key[[k]]; nm <- if (length(v) > 1) paste0(k, "_", seq_along(v)) else k
  setNames(as.numeric(v), nm)
}))
write.csv(data.frame(name = names(flat), value = sprintf("%.10g", flat)), "cache/ch13_lab_R.csv", row.names = FALSE)
cat("wrote", length(flat), "numbers to cache/ch13_lab_R.csv\n")
