# Chapter 8. High-Throughput Count Data and Generalized Linear Models
# Course: Statistical Methods for Modern Biology, following Holmes and Huber (2019),
# Modern Statistics for Modern Biology, Cambridge University Press, Chapter 8.
#
# Reproduces every figure of the chapter that is computed from data or by simulation:
#   Figures 8.1 to 8.14 (all fourteen). Figure 8.10 has two panels (a) and (b), drawn side by side.
# Skipped figures: none (the chapter has no photographs or schematics that are numbered figures).
# Supplementary figures for the notes (not in the book):
#   fig-8-s1  gene-wise, trend and shrunken (MAP) dispersion estimates for the pasilla data
#   fig-8-s2  paired versus unpaired analysis of the airway data (cell line as a blocking factor)
#
# Data: the pasilla package (count table and sample annotation) and the airway package.
# The script also defines the course's own implementations (median-of-ratios size factors, the
# negative binomial GLM fitted by IRLS, the Cox-Reid adjusted dispersion likelihood with and
# without the log-normal prior, the parametric dispersion trend, the closed-form VST, rlog for one
# gene, Huber M-estimation), each the counterpart of a DESeq2 / MASS function.
#
# Run:  Rscript ch08_count_data_glm.R            (all figures, exports the demo data)
#       Rscript ch08_count_data_glm.R fig_8_5    (one figure)

this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1])))
  else if (!is.null(sys.frame(1)$ofile)) dirname(normalizePath(sys.frame(1)$ofile))
  else getwd()
})
source(file.path(this_dir, "common.R"))

suppressPackageStartupMessages({
  library(DESeq2)
  library(dplyr)
  library(tibble)
})

CH <- 8
CACHE <- file.path(CODE_DIR, "cache")
dir.create(CACHE, showWarnings = FALSE)
cached <- function(name, expr) {
  f <- file.path(CACHE, sprintf("ch08-%s.rds", name))
  if (file.exists(f)) return(readRDS(f))
  val <- force(expr)
  saveRDS(val, f)
  val
}

# ================================================================================================
# Course implementations (each the counterpart of a DESeq2 / MASS function)
# ================================================================================================

# Median-of-ratios size factors (Anders and Huber 2010): genes with a zero in any sample have
# geometric mean 0 and are left out; s_j = median_i K_ij / (prod_j' K_ij')^(1/m).
sf_median_ratio <- function(K) {
  lg <- rowMeans(log(K))
  ok <- is.finite(lg)
  apply(K[ok, , drop = FALSE], 2, function(k) exp(median(log(k) - lg[ok])))
}

# Gamma-Poisson (negative binomial) log pmf in the mean-dispersion form: mean mu, variance
# mu + alpha mu^2. Written out from the gamma functions rather than calling dnbinom.
ldgp <- function(k, mu, alpha) {
  r <- 1 / alpha
  lgamma(k + r) - lgamma(r) - lgamma(k + 1) + r * log(r / (r + mu)) + k * log(mu / (r + mu))
}
nb_loglik <- function(k, mu, alpha) sum(ldgp(k, mu, alpha))

# IRLS for the NB GLM  log mu_j = log s_j + x_j' b  (b on the natural-log scale internally).
# lambda2: ridge penalty on the log2-scale coefficients (DESeq2 uses 1e-6 for an MLE, and
# 1 / betaPriorVar for the shrunken fold change). Returns log2-scale coefficients and SEs.
nb_irls <- function(k, X, s, alpha, lambda2 = rep(1e-6, ncol(X)), maxit = 100, tol = 1e-8,
                    beta0 = NULL) {
  lam <- lambda2 / log(2)^2
  b <- if (is.null(beta0)) qr.solve(X, log(k / s + 0.1)) else beta0 * log(2)
  path <- list(b); ll <- c(); dev_old <- Inf
  for (it in seq_len(maxit)) {
    eta <- drop(X %*% b) + log(s); mu <- exp(eta)
    w <- mu / (1 + alpha * mu)                       # IRLS weight 1 / (V(mu) g'(mu)^2)
    z <- eta - log(s) + (k - mu) / mu                # working response (offset removed)
    A <- crossprod(X, w * X) + diag(lam, ncol(X))
    b <- drop(solve(A, crossprod(X, w * z)))
    mu <- exp(drop(X %*% b) + log(s))
    dev <- -2 * nb_loglik(k, mu, alpha)
    path[[it + 1]] <- b; ll <- c(ll, -dev / 2)
    if (!is.finite(dev)) break                       # diverged (e.g. a poor starting value)
    if (abs(dev - dev_old) / (abs(dev) + 0.1) < tol) break
    dev_old <- dev
  }
  w <- mu / (1 + alpha * mu)
  I <- crossprod(X, w * X); A <- I + diag(lam, ncol(X)); Ai <- solve(A)
  V <- Ai %*% I %*% Ai                               # sandwich form used by DESeq2
  list(beta = b / log(2), se = sqrt(diag(V)) / log(2), mu = mu, loglik = -dev / 2, iter = it,
       path = do.call(rbind, path) / log(2), ll = ll, w = w)
}

# Cox-Reid adjusted profile log likelihood of log(alpha), with the fitted means mu held fixed.
cr_apl <- function(la, k, X, mu) {
  a <- exp(la); w <- mu / (1 + a * mu)
  nb_loglik(k, mu, a) - 0.5 * determinant(crossprod(X, w * X), logarithm = TRUE)$modulus[1]
}
# Gene-wise estimate (no prior) or MAP estimate with the log-normal prior N(log prior_mean, prior_var).
disp_est <- function(k, X, mu, prior_mean = NULL, prior_var = NULL, lo = log(1e-8), hi = log(10)) {
  f <- function(la) {
    v <- cr_apl(la, k, X, mu)
    if (!is.null(prior_mean)) v <- v - (la - log(prior_mean))^2 / (2 * prior_var)
    v
  }
  exp(optimize(f, c(lo, hi), maximum = TRUE, tol = 1e-10)$maximum)
}

# Parametric dispersion trend alpha(mu) = a0 + a1 / mu: gamma-family GLM with identity link,
# refitted after dropping genes whose ratio to the current fit is outside (1e-4, 15).
disp_trend <- function(means, disps) {
  coefs <- c(0.1, 1)
  for (iter in 1:10) {
    ratio <- disps / (coefs[1] + coefs[2] / means)
    good <- which(ratio > 1e-4 & ratio < 15)
    fit <- glm(disps[good] ~ I(1 / means[good]), family = Gamma(link = "identity"), start = coefs)
    old <- coefs; coefs <- unname(coef(fit))
    if (sum(log(coefs / old)^2) < 1e-6) break
  }
  c(a0 = coefs[1], a1 = coefs[2])
}
# Prior variance of log dispersions: robust spread of log(gene-wise / trend) minus the sampling
# variance trigamma((m - p) / 2) of a log dispersion estimate, floored at 0.25.
disp_prior_var <- function(gene, fit, m, p) {
  r <- log(gene) - log(fit)
  max(mad(r)^2 - trigamma((m - p) / 2), 0.25)
}

# Closed-form variance-stabilising transformation for alpha(mu) = a0 + a1 / mu (log2 scale).
vst_param <- function(q, a0, a1) {
  log2((1 + a1 + 2 * a0 * q + 2 * sqrt(a0 * q * (1 + a1 + a0 * q))) / (4 * a0))
}

# Upper-quantile matching for a normal prior variance (used for the LFC and rlog priors): the
# weighted (1 - upper) quantile q of |x| is set equal to the same quantile of |N(0, sigma^2)|,
# sigma = q / z_{1 - upper/2}. The weighted quantile interpolates between order statistics of the
# weight-normalised empirical distribution (weights rescaled to sum to n).
weighted_upper_var <- function(x, w, upper = 0.05) {
  ax <- abs(x); n <- length(ax); w <- w * n / sum(w)
  xs <- sort(unique(ax)); cw <- cumsum(as.numeric(tapply(w, match(ax, xs), sum)))
  ord <- 1 + (n - 1) * (1 - upper); lo <- max(floor(ord), 1); hi <- min(lo + 1, n); fr <- ord %% 1
  qv <- approx(cw, xs, xout = c(lo, hi), method = "constant", f = 1, rule = 2)$y
  q <- (1 - fr) * qv[1] + fr * qv[2]
  (q / qnorm(1 - upper / 2))^2
}

# Huber M-estimation by iteratively reweighted least squares, scale re-estimated by the MAD of the
# residuals at every step (the algorithm of MASS::rlm with psi.huber, k = 1.345).
huber_rho <- function(x, s) ifelse(abs(x) < s, x^2 / 2, s * abs(x) - s^2 / 2)
huber_irls <- function(X, y, k = 1.345, maxit = 50, tol = 1e-4) {
  b <- qr.solve(X, y); r <- drop(y - X %*% b)
  for (it in seq_len(maxit)) {
    s <- median(abs(r)) / 0.6745
    u <- r / s
    w <- pmin(1, k / abs(u))
    b <- drop(solve(crossprod(X, w * X), crossprod(X, w * y)))
    rn <- drop(y - X %*% b)
    conv <- sqrt(sum((rn - r)^2) / max(1e-20, sum(rn^2)))
    r <- rn
    if (conv < tol) break
  }
  list(coef = b, scale = s, w = w, iter = it)
}

# ================================================================================================
# Data
# ================================================================================================
pasilla_counts <- function() {
  fn <- system.file("extdata", "pasilla_gene_counts.tsv", package = "pasilla", mustWork = TRUE)
  as.matrix(read.csv(fn, sep = "\t", row.names = "gene_id"))
}
pasilla_anno <- function(K) {
  af <- system.file("extdata", "pasilla_sample_annotation.csv", package = "pasilla", mustWork = TRUE)
  a <- read.csv(af, check.names = FALSE)
  a$condition <- factor(a$condition, levels = c("untreated", "treated"))
  a$type <- factor(sub("-.*", "", a$type), levels = c("single", "paired"))
  a <- a[match(colnames(K), sub("fb$", "", a$file)), c("file", "condition", "type")]
  stopifnot(!anyNA(a$condition))
  rownames(a) <- colnames(K)
  a
}

# All the DESeq2 fits used by the figures, run step by step so that the intermediate fitted means
# of the gene-wise dispersion step are kept (they are needed to reproduce the MAP step exactly).
pasilla_fits <- function() cached("pasilla", {
  K <- pasilla_counts(); cd <- pasilla_anno(K)
  run <- function(design) {
    d <- DESeqDataSetFromMatrix(K, cd, design)
    d <- estimateSizeFactors(d)
    d <- estimateDispersionsGeneEst(d)
    mu_ge <- assays(d)[["mu"]]
    d <- estimateDispersionsFit(d)
    d <- estimateDispersionsMAP(d)
    d <- nbinomWaldTest(d)
    list(dds = d, mu_ge = mu_ge)
  }
  one <- run(~ condition)
  two <- run(~ type + condition)
  res1 <- results(one$dds)
  res2 <- results(two$dds)
  resType <- results(two$dds, contrast = c("type", "single", "paired"))
  lrt2 <- results(nbinomLRT(two$dds, reduced = ~ type))
  r1nc <- results(one$dds, cooksCutoff = FALSE)
  shr <- lfcShrink(one$dds, coef = "condition_treated_vs_untreated", type = "normal", res = r1nc,
                   quiet = TRUE)
  thr <- lapply(setNames(nm = c("greaterAbs", "lessAbs", "greater", "less")), function(h)
    results(one$dds, lfcThreshold = 0.5, altHypothesis = h))
  vsd <- varianceStabilizingTransformation(one$dds, blind = TRUE)
  rld <- rlog(one$dds, blind = TRUE)
  blind <- one$dds; design(blind) <- ~ 1
  blind <- estimateDispersions(blind, quiet = TRUE)
  list(K = K, cd = cd, one = one, two = two, res1 = res1, res2 = res2, resType = resType,
       lrt2 = lrt2, r1nc = r1nc, shr = shr, thr = thr, vsd = vsd, rld = rld,
       blind_coef = attr(dispersionFunction(blind), "coefficients"),
       blind_dispFit = mcols(blind)$dispFit)
})

airway_fits <- function() cached("airway", {
  suppressPackageStartupMessages(library(airway))
  data("airway", package = "airway", envir = environment())
  a <- airway
  a$dex <- relevel(a$dex, "untrt")
  a <- a[rowSums(assay(a)) >= 10, ]
  paired <- DESeq(DESeqDataSet(a, ~ cell + dex), quiet = TRUE)
  unpaired <- DESeq(DESeqDataSet(a, ~ dex), quiet = TRUE)
  list(cd = as.data.frame(colData(a)[, c("cell", "dex")]), n = nrow(a),
       rp = as.data.frame(results(paired)), ru = as.data.frame(results(unpaired)))
})

# Exercise 8.2: the one-factor analysis with edgeR (quasi-likelihood F-test), for comparison with DESeq2.
edger_compare <- function() cached("edger", {
  F <- pasilla_fits()
  y <- edgeR::DGEList(F$K, group = F$cd$condition)
  keep <- edgeR::filterByExpr(y)
  y <- edgeR::normLibSizes(y[keep, , keep.lib.sizes = FALSE])
  X <- model.matrix(~ condition, F$cd)
  y <- edgeR::estimateDisp(y, X)
  fit <- edgeR::glmQLFit(y, X)
  qlf <- edgeR::glmQLFTest(fit, coef = 2)
  tt <- edgeR::topTags(qlf, n = Inf, sort.by = "none")$table
  list(table = tt, common_disp = y$common.dispersion)
})

trsf6 <- function(x) ifelse(is.na(x), 0, (-log10(x))^(1 / 6))

# ================================================================================================
# Figures
# ================================================================================================

# Figure 8.1: toy example of size factor estimation (five genes, two samples).
toy_sf <- function() {
  d <- data.frame(x = c(2, 4, 6, 6, 8) * 10, y = c(3, 6, 2, 9, 12) * 10, name = LETTERS[1:5])
  sf <- sf_median_ratio(as.matrix(d[, c("x", "y")]))
  list(d = d, slope_sum = sum(d$y) / sum(d$x), slope_mr = unname(sf[2] / sf[1]))
}
fig_8_1 <- function() {
  t <- toy_sf()
  lines <- data.frame(slope = c(t$slope_sum, t$slope_mr),
                      method = factor(c("ratio of column sums", "median of ratios"),
                                      levels = c("ratio of column sums", "median of ratios")))
  p <- ggplot(t$d, aes(x, y)) +
    geom_abline(data = lines, aes(slope = slope, intercept = 0, colour = method), linewidth = 0.8) +
    geom_point(size = 2.2) + geom_text(aes(label = name), vjust = -0.8, size = 3.6) +
    scale_colour_manual(values = unname(COL[c("blue", "red")]), name = NULL) +
    coord_fixed(xlim = c(0, 128), ylim = c(0, 128), expand = FALSE) +
    labs(x = "sample 1", y = "sample 2") + theme(legend.position = "bottom")
  savefig(p, "fig-8-1", ch = CH, width = 4.2, height = 4.6)
}

# Figure 8.2: median-of-ratios size factors against library sizes (column sums).
fig_8_2 <- function() {
  F <- pasilla_fits()
  d <- tibble(sf = sf_median_ratio(F$K), total = colSums(F$K) / 1e6, sample = colnames(F$K),
              type = F$cd$type)
  fit <- lm(total ~ 0 + sf, d)
  p <- ggplot(d, aes(sf, total)) +
    geom_abline(slope = coef(fit), intercept = 0, colour = "grey60", linetype = 2) +
    geom_point(aes(colour = type), size = 2.4) +
    ggrepel::geom_text_repel(aes(label = sample), size = 2.7, seed = 8, min.segment.length = 0.2) +
    scale_colour_manual(values = unname(COL[c("orange", "teal")])) +
    scale_x_continuous(expand = expansion(mult = 0.12)) +
    labs(x = "size factor (median of ratios)", y = "column sum (millions of reads)", colour = "library type")
  savefig(p, "fig-8-2", ch = CH, width = 5.4, height = 3.8)
}

# Figure 8.3: variance against mean for the four untreated replicates (normalised counts).
fig_8_3 <- function() {
  F <- pasilla_fits()
  sf <- sf_median_ratio(F$K)
  nc <- sweep(F$K, 2, sf, "/")[, F$cd$condition == "untreated"]
  d <- tibble(m = rowMeans(nc), v = matrixStats::rowVars(nc)) |> filter(m > 0, v > 0)
  co <- attr(dispersionFunction(F$one$dds), "coefficients")   # trend of the ~ condition fit
  grid <- tibble(m = exp(seq(log(min(d$m)), log(max(d$m)), length.out = 200))) |>
    mutate(v = m + (co[1] + co[2] / m) * m^2)
  p <- ggplot(d, aes(log(m), log(v))) +
    geom_hex(bins = 60) + scale_fill_gradient(low = "grey80", high = "grey10", trans = "log10") +
    geom_abline(slope = 1, intercept = 0, colour = COL[["green"]], linewidth = 0.8) +
    geom_abline(slope = 2, intercept = 0, colour = COL[["red"]], linewidth = 0.8) +
    geom_line(data = grid, colour = COL[["blue"]], linewidth = 0.8, linetype = 2) +
    coord_fixed() + theme(legend.position = "none") +
    labs(x = "log(mean)", y = "log(variance)")
  savefig(p, "fig-8-3", ch = CH, width = 4.4, height = 4.6)
}

# Figure 8.4: histogram of the Wald test p-values (one-factor design).
fig_8_4 <- function() {
  F <- pasilla_fits()
  d <- tibble(p = F$res1$pvalue) |> filter(!is.na(p))
  bg <- median(hist(d$p, breaks = seq(0, 1, 0.01), plot = FALSE)$counts)
  p <- ggplot(d, aes(p)) +
    geom_histogram(binwidth = 0.01, boundary = 0, fill = COL[["blue"]], colour = NA) +
    geom_hline(yintercept = bg, colour = COL[["orange"]], linetype = 2) +
    labs(x = "p-value", y = "number of genes")
  savefig(p, "fig-8-4", ch = CH, width = 5, height = 3.4)
}

# MA plot helper: mean of normalised counts (log scale) against log2 fold change, values outside
# the y range drawn as triangles at the edge, significant genes coloured.
ma_data <- function(res, ylim) {
  as.data.frame(res) |> filter(baseMean > 0) |>
    mutate(sig = !is.na(padj) & padj < 0.1,
           out = log2FoldChange < ylim[1] | log2FoldChange > ylim[2],
           y = pmin(pmax(log2FoldChange, ylim[1]), ylim[2]))
}
ma_plot <- function(d, ylim) {
  ggplot(d, aes(baseMean, y)) +
    geom_point(data = filter(d, !sig), colour = "grey55", size = 0.35,
               aes(shape = out)) +
    geom_point(data = filter(d, sig), colour = COL[["red"]], size = 0.45, aes(shape = out)) +
    geom_hline(yintercept = 0, colour = "grey30") +
    scale_shape_manual(values = c(`FALSE` = 16, `TRUE` = 17), guide = "none") +
    scale_x_log10() + coord_cartesian(ylim = ylim) +
    labs(x = "mean of normalised counts", y = "log2 fold change")
}

# Figure 8.5: MA plot of the one-factor analysis.
fig_8_5 <- function() {
  F <- pasilla_fits()
  p <- ma_plot(ma_data(F$res1, c(-2, 2)), c(-2, 2))
  savefig(p, "fig-8-5", ch = CH, width = 5.6, height = 4, dense = TRUE)
}

# PCA of the 500 most variable rlog-transformed genes (as plotPCA does).
rlog_pca <- function(rld, ntop = 500) {
  x <- assay(rld)
  sel <- order(matrixStats::rowVars(x), decreasing = TRUE)[seq_len(ntop)]
  pc <- prcomp(t(x[sel, ]))
  list(scores = pc$x, pve = pc$sdev^2 / sum(pc$sdev^2))
}
# Figure 8.6: PCA plot of the seven samples.
fig_8_6 <- function() {
  F <- pasilla_fits()
  P <- rlog_pca(F$rld)
  d <- tibble(PC1 = P$scores[, 1], PC2 = P$scores[, 2], condition = F$cd$condition, type = F$cd$type)
  p <- ggplot(d, aes(PC1, PC2, colour = condition, shape = type)) + geom_point(size = 3) +
    scale_colour_manual(values = unname(COL[c("blue", "red")])) +
    coord_fixed() +
    labs(x = sprintf("PC1: %.0f%% of variance", 100 * P$pve[1]),
         y = sprintf("PC2: %.0f%% of variance", 100 * P$pve[2]))
  savefig(p, "fig-8-6", ch = CH, width = 6, height = 3.4)
}

# Figure 8.7: heatmap of the 30 genes with the highest mean rlog value, rows centred and scaled.
fig_8_7 <- function() {
  F <- pasilla_fits()
  x <- assay(F$rld)
  sel <- order(rowMeans(x), decreasing = TRUE)[1:30]
  ann <- data.frame(condition = F$cd$condition, type = F$cd$type, row.names = colnames(x))
  cols <- list(condition = c(untreated = COL[["blue"]], treated = COL[["red"]]),
               type = c(single = COL[["orange"]], paired = COL[["teal"]]))
  ph <- pheatmap::pheatmap(x[sel, ], scale = "row", annotation_col = ann, annotation_colors = cols,
                           fontsize = 8, fontsize_row = 7, silent = TRUE)
  savefig(ph$gtable, "fig-8-7", ch = CH, width = 5, height = 7)
}

# Figure 8.8: Huber's loss with s = 2 against the parabola.
fig_8_8 <- function() {
  x <- seq(-7, 7, length.out = 281)
  d <- bind_rows(tibble(x = x, value = x^2 / 2, loss = "parabola"),
                 tibble(x = x, value = huber_rho(x, 2), loss = "Huber, s = 2"))
  p <- ggplot(d, aes(x, value, colour = loss)) + geom_line(linewidth = 0.9) +
    geom_vline(xintercept = c(-2, 2), colour = "grey60", linetype = 3) +
    scale_colour_manual(values = unname(COL[c("blue", "orange")]), name = NULL) +
    labs(x = expression(epsilon), y = expression(rho[s](epsilon))) +
    theme(legend.position = c(0.5, 0.85))
  savefig(p, "fig-8-8", ch = CH, width = 4.6, height = 3.4)
}

# Figure 8.9: p-values of the one-factor against the two-factor analysis.
fig_8_9 <- function() {
  F <- pasilla_fits()
  d <- tibble(a = trsf6(F$res1$pvalue), b = trsf6(F$res2$pvalue))
  p <- ggplot(d, aes(a, b)) + geom_hex(bins = 75) +
    scale_fill_viridis_c(trans = "log10", name = "genes") +
    geom_abline(colour = COL[["orange"]], linewidth = 0.8) + coord_fixed() +
    labs(x = "single factor (condition)", y = "two factors (type + condition)")
  savefig(p, "fig-8-9", ch = CH, width = 5, height = 4.3)
}

# Figure 8.10: likelihood, prior and posterior of the log2 fold change for two genes.
FIG810_GENES <- c("FBgn0053926", "FBgn0260933")
shrink_curves <- function(F, genes = FIG810_GENES, grid = seq(-1, 1.5, length.out = 501)) {
  d <- F$one$dds; K <- counts(d); s <- sizeFactors(d)
  cond <- as.numeric(d$condition == "treated")
  bpv <- priorInfo(F$shr)$betaPriorVar[2]
  prior <- dnorm(grid, 0, sqrt(bpv))
  out <- list(); h <- diff(grid[1:2])
  for (g in genes) {
    a <- dispersions(d)[rownames(d) == g]
    b0 <- mcols(d)[g, "Intercept"]
    ll <- vapply(grid, function(b) nb_loglik(K[g, ], s * 2^(b0 + b * cond), a), 0)
    lik <- exp(ll - max(ll)); lik <- lik / (sum(lik) * h)
    post <- lik * prior; post <- post / (sum(post) * h)
    out[[g]] <- tibble(beta = grid, likelihood = lik, posterior = post, gene = g)
  }
  list(curves = bind_rows(out), prior = tibble(beta = grid, prior = prior), bpv = bpv)
}
fig_8_10 <- function() {
  F <- pasilla_fits()
  d <- F$one$dds
  gc <- c(FBgn0053926 = COL[["green"]], FBgn0260933 = COL[["blue"]])
  da <- tibble(k = as.vector(t(counts(d, normalized = TRUE)[FIG810_GENES, ])),
               condition = rep(d$condition, 2), gene = rep(FIG810_GENES, each = ncol(d)))
  pa <- ggplot(da, aes(condition, k, colour = gene)) +
    ggbeeswarm::geom_beeswarm(cex = 3, size = 2) + facet_grid(. ~ gene) +
    scale_y_log10() + scale_colour_manual(values = gc) + theme(legend.position = "none") +
    labs(x = NULL, y = "normalised counts", title = "(a)")
  S <- shrink_curves(F)
  cl <- S$curves |> tidyr::pivot_longer(c(likelihood, posterior), names_to = "what", values_to = "y")
  mx <- cl |> group_by(gene, what) |> slice_max(y, n = 1, with_ties = FALSE)
  pb <- ggplot(cl, aes(beta, y, colour = gene, linetype = what)) + geom_line(linewidth = 0.8) +
    geom_line(data = S$prior, aes(beta, prior), inherit.aes = FALSE, colour = "black") +
    geom_point(data = mx, size = 1.8) +
    scale_colour_manual(values = gc) +
    scale_linetype_manual(values = c(likelihood = "solid", posterior = "22")) +
    labs(x = expression(beta ~ "(log2 fold change)"), y = "density", title = "(b)",
         colour = NULL, linetype = NULL) +
    theme(legend.position = "bottom", legend.box = "vertical", legend.margin = margin(0, 0, 0, 0))
  g <- gridExtra::arrangeGrob(pa, pb, ncol = 2, widths = c(1, 1.15))
  savefig(g, "fig-8-10", ch = CH, width = 8.4, height = 3.9)
}

# Figure 8.11: the variance-stabilising transformation and log2 for sample 1.
fig_8_11 <- function() {
  F <- pasilla_fits()
  k <- counts(F$one$dds)[, 1]
  d <- bind_rows(tibble(counts = k, y = assay(F$vsd)[, 1], transformation = "VST"),
                 tibble(counts = k, y = log2(k), transformation = "log2")) |>
    filter(counts <= 600, is.finite(y)) |> arrange(transformation, counts)
  p <- ggplot(d, aes(counts, y, colour = transformation)) + geom_line(linewidth = 0.8) +
    scale_colour_manual(values = unname(COL[c("orange", "blue")]), name = NULL) +
    coord_cartesian(xlim = c(0, 600), ylim = c(0, 9.5)) +
    labs(x = "counts (sample untreated1)", y = "transformed value") +
    theme(legend.position = c(0.8, 0.25))
  savefig(p, "fig-8-11", ch = CH, width = 4.6, height = 3.6)
}

# Figure 8.12: per-gene sd against the rank of the mean, for log2(n + 1), VST and rlog.
fig_8_12 <- function() {
  F <- pasilla_fits()
  mats <- list(`log2(n + 1)` = log2(counts(F$one$dds, normalized = TRUE) + 1),
               vst = assay(F$vsd), rlog = assay(F$rld))
  d <- bind_rows(lapply(names(mats), function(nm) {
    x <- mats[[nm]]
    tibble(rank = rank(rowMeans(x), ties.method = "first"), sd = matrixStats::rowSds(x), what = nm)
  })) |> mutate(what = factor(what, levels = names(mats)))
  run <- d |> group_by(what) |> arrange(rank, .by_group = TRUE) |>
    mutate(med = stats::runmed(sd, 501, endrule = "median")) |> ungroup()
  p <- ggplot(d, aes(rank, sd)) +
    geom_hex(bins = 50) + scale_fill_gradient(low = "grey85", high = "grey15", trans = "log10") +
    geom_line(data = run, aes(rank, med), colour = COL[["red"]], linewidth = 0.8) +
    facet_wrap(~ what, nrow = 1) + coord_cartesian(ylim = c(0, 1.5)) +
    theme(legend.position = "none") + labs(x = "rank of the mean", y = "standard deviation")
  savefig(p, "fig-8-12", ch = CH, width = 9, height = 3.4)
}

# Figure 8.13: MA plots for the four threshold tests, theta = 0.5.
fig_8_13 <- function() {
  F <- pasilla_fits()
  lab <- c(greaterAbs = "greaterAbs: |beta| > 0.5", lessAbs = "lessAbs: |beta| < 0.5",
           greater = "greater: beta > 0.5", less = "less: beta < -0.5")
  d <- bind_rows(lapply(names(F$thr), function(h)
    ma_data(F$thr[[h]], c(-2.5, 2.5)) |> mutate(test = lab[[h]]))) |>
    mutate(test = factor(test, levels = lab))
  hl <- tibble(test = factor(lab[c(1, 1, 2, 2, 3, 4)], levels = lab), y = c(-.5, .5, -.5, .5, .5, -.5))
  p <- ma_plot(d, c(-2.5, 2.5)) +
    geom_hline(data = hl, aes(yintercept = y), colour = COL[["teal"]], linewidth = 0.7) +
    facet_wrap(~ test, ncol = 1)
  savefig(p, "fig-8-13", ch = CH, width = 4.4, height = 9.4, dense = TRUE)
}

# Exercise 8.1 simulation (Figure 8.14): no group effect, x2 has a balanced batch shift.
batch_sim <- function(ng = 10000, ns = 12, seed = 8) {
  set.seed(seed)
  x1 <- matrix(rnorm(ns * ng), nrow = ng)
  group <- factor(letters[1 + seq_len(ns) %% 2])
  batch <- factor(ifelse(seq_len(ns) <= ns / 2, "B1", "B2"))
  x2 <- x1
  x2[, batch == "B2"] <- x2[, batch == "B2"] + 2 * rnorm(ng)
  list(x1 = x1, x2 = x2, group = group, batch = batch)
}
# p-values for the group coefficient of a gene-by-gene linear model (vectorised least squares).
lm_pvalues <- function(Y, X, coef) {
  XtXi <- solve(crossprod(X)); B <- Y %*% X %*% XtXi
  R <- Y - B %*% t(X); df <- nrow(X) - ncol(X)
  s2 <- rowSums(R^2) / df
  tstat <- B[, coef] / sqrt(s2 * XtXi[coef, coef])
  2 * pt(-abs(tstat), df)
}
fig_8_14 <- function() {
  S <- batch_sim()
  Xg <- model.matrix(~ S$group); Xgb <- model.matrix(~ S$group + S$batch)
  d <- bind_rows(
    tibble(p = lm_pvalues(S$x1, Xg, 2), data = "x1", model = "~ group (t-test)"),
    tibble(p = lm_pvalues(S$x2, Xg, 2), data = "x2", model = "~ group (t-test)"),
    tibble(p = lm_pvalues(S$x1, Xgb, 2), data = "x1", model = "~ group + batch"),
    tibble(p = lm_pvalues(S$x2, Xgb, 2), data = "x2", model = "~ group + batch"))
  p <- ggplot(d, aes(p)) + geom_histogram(binwidth = 0.02, boundary = 0, fill = COL[["blue"]]) +
    facet_grid(data ~ model) + labs(x = "p-value for the group coefficient", y = "number of genes")
  savefig(p, "fig-8-14", ch = CH, width = 6.4, height = 4.2)
}

# Supplementary figure s1: dispersion estimates.
fig_8_s1 <- function() {
  F <- pasilla_fits()
  m <- as.data.frame(mcols(F$one$dds)) |> filter(!allZero, baseMean > 0)
  co <- attr(dispersionFunction(F$one$dds), "coefficients")
  d <- bind_rows(tibble(mean = m$baseMean, disp = m$dispGeneEst, what = "gene-wise (Cox-Reid MLE)"),
                 tibble(mean = m$baseMean, disp = m$dispersion, what = "final (MAP, outliers kept)")) |>
    filter(disp > 1e-7)
  tr <- tibble(mean = 10^seq(-1, 5.3, length.out = 200)) |> mutate(disp = co[1] + co[2] / mean)
  p <- ggplot(d, aes(mean, disp)) +
    geom_point(aes(colour = what), size = 0.3, alpha = 0.5) +
    geom_line(data = tr, colour = COL[["red"]], linewidth = 0.9) +
    scale_colour_manual(values = c(`gene-wise (Cox-Reid MLE)` = "grey20",
                                   `final (MAP, outliers kept)` = COL[["blue"]]), name = NULL) +
    guides(colour = guide_legend(override.aes = list(size = 2, alpha = 1))) +
    scale_x_log10() + scale_y_log10() + coord_cartesian(ylim = c(1e-4, 20)) +
    labs(x = "mean of normalised counts", y = "dispersion") + theme(legend.position = "bottom")
  savefig(p, "fig-8-s1", ch = CH, width = 5.6, height = 4.6, dense = TRUE)
}

# Supplementary figure s2: airway, paired (~ cell + dex) against unpaired (~ dex) p-values.
fig_8_s2 <- function() {
  A <- airway_fits()
  d <- tibble(a = trsf6(A$ru$pvalue), b = trsf6(A$rp$pvalue))
  p <- ggplot(d, aes(a, b)) + geom_hex(bins = 70) +
    scale_fill_viridis_c(trans = "log10", name = "genes") +
    geom_abline(colour = COL[["orange"]], linewidth = 0.8) + coord_fixed() +
    labs(x = "unpaired analysis (~ dex)", y = "paired analysis (~ cell + dex)")
  savefig(p, "fig-8-s2", ch = CH, width = 5, height = 4.3)
}

# ================================================================================================
# Data for the browser demonstrations
# ================================================================================================
export_demo_data <- function() {
  F <- pasilla_fits()
  d1 <- F$one$dds; d2 <- F$two$dds
  K <- counts(d1)
  # (1) size factors and the mean-variance relationship: genes without zeros
  pos <- K[apply(K > 0, 1, all), ]
  export_json(list(samples = colnames(K), condition = as.character(F$cd$condition),
                   type = as.character(F$cd$type),
                   counts = unname(lapply(seq_len(ncol(pos)), function(j) unname(pos[, j]))),
                   sf_deseq2 = unname(sizeFactors(d1)), colsums = unname(colSums(K)),
                   trend = unname(attr(dispersionFunction(d1), "coefficients")),
                   trend_blind = unname(F$blind_coef), toy = toy_sf()$d[, c("x", "y")]),
              "ch08-sizefactors.json", digits = 8)
  # (2) one gene at a time: counts and the DESeq2 reference numbers
  r1 <- F$res1; r2 <- F$res2
  top <- rownames(r1)[order(r1$padj)][1:8]
  set.seed(8)
  mid <- sample(rownames(r1)[which(r1$baseMean > 50 & r1$padj > 0.3)], 6)
  low <- sample(rownames(r1)[which(r1$baseMean > 3 & r1$baseMean < 12 & !is.na(r1$pvalue))], 4)
  twof <- rownames(r2)[order(-(abs(r2$stat) - abs(r1$stat)))][1:4]
  # two low-count genes where the Wald and likelihood ratio tests disagree (found by comparing both
  # statistics over low-count genes): FBgn0037191 (skewed likelihood) and FBgn0053498 (all treated
  # counts zero, so the MLE runs off to minus infinity and the Wald statistic collapses)
  genes <- unique(c(FIG810_GENES, top, twof, "FBgn0037191", "FBgn0053498", mid, low))
  bpv <- priorInfo(F$shr)$betaPriorVar[2]
  gl <- lapply(genes, function(g) list(
    id = g, k = unname(K[g, ]),
    disp1 = unname(dispersions(d1)[rownames(d1) == g]), disp2 = unname(dispersions(d2)[rownames(d2) == g]),
    lfc1 = r1[g, "log2FoldChange"], se1 = r1[g, "lfcSE"], stat1 = r1[g, "stat"], p1 = r1[g, "pvalue"],
    lfc2 = r2[g, "log2FoldChange"], se2 = r2[g, "lfcSE"], stat2 = r2[g, "stat"], p2 = r2[g, "pvalue"],
    lrt2 = F$lrt2[g, "stat"], shr = F$shr[g, "log2FoldChange"], baseMean = r1[g, "baseMean"]))
  export_json(list(sf = unname(sizeFactors(d1)), condition = as.character(F$cd$condition),
                   type = as.character(F$cd$type), samples = colnames(K), betaPriorVar = unname(bpv),
                   genes = gl), "ch08-genes.json", digits = 8)
  # (3) dispersion shrinkage: a stratified sample of genes with counts and the gene-wise-step means
  m <- mcols(d1)
  ok <- which(!m$allZero & m$baseMean > 1 & m$dispGeneEst > 1e-6)
  set.seed(8)
  br <- cut(log10(m$baseMean[ok]), 12)
  pick <- unlist(lapply(split(ok, br), function(ix) if (length(ix) > 110) sample(ix, 110) else ix))
  pick <- sort(pick)
  mu <- F$one$mu_ge
  export_json(list(sf = unname(sizeFactors(d1)), condition = as.character(F$cd$condition),
                   trend = unname(attr(dispersionFunction(d1), "coefficients")),
                   priorVar = unname(attr(dispersionFunction(d1), "dispPriorVar")),
                   blindTrend = unname(F$blind_coef),
                   baseMean = unname(m$baseMean[pick]), gene = unname(m$dispGeneEst[pick]),
                   fit = unname(m$dispFit[pick]), map = unname(m$dispMAP[pick]),
                   final = unname(m$dispersion[pick]),
                   k = unname(lapply(pick, function(i) unname(K[i, ]))),
                   mu = unname(lapply(pick, function(i) signif(unname(mu[i, ]), 10)))),
              "ch08-dispersion.json", digits = 10)
}

if (sys.nframe() == 0L) {
  run_figures(list(fig_8_1 = fig_8_1, fig_8_2 = fig_8_2, fig_8_3 = fig_8_3, fig_8_4 = fig_8_4,
                   fig_8_5 = fig_8_5, fig_8_6 = fig_8_6, fig_8_7 = fig_8_7, fig_8_8 = fig_8_8,
                   fig_8_9 = fig_8_9, fig_8_10 = fig_8_10, fig_8_11 = fig_8_11, fig_8_12 = fig_8_12,
                   fig_8_13 = fig_8_13, fig_8_14 = fig_8_14, fig_8_s1 = fig_8_s1, fig_8_s2 = fig_8_s2,
                   export = export_demo_data))
}
