# Chapter 4. Mixture Models
# Course: Statistical Methods for Modern Biology (following Holmes and Huber 2019,
# Modern Statistics for Modern Biology, Cambridge University Press, Chapter 4).
#
# Reproduces every figure of the chapter that is computed from data or by simulation:
#   Figure 4.1   10,000 draws from a fair mixture of N(1, 0.5^2) and N(3, 0.5^2)
#   Figure 4.2   the same mixture with one million draws and 200 bins
#   Figure 4.3   the first component alone, with the normal density on top
#   Figure 4.4   the theoretical mixture density (Equation 4.1)
#   Figure 4.5   a harder mixture: N(1, 0.5) and N(2, 0.5), 1,000 draws
#   Figure 4.6   the same, components overlaid in two colours
#   Figure 4.7   the same, components stacked
#   Figure 4.8   ChIP-Seq tag counts per 200 nt bin, chromosome 22 (mosaicsExample data)
#   Figure 4.9   the same histogram on a log10 count scale
#   Figure 4.10  7,000 simulated nucleotide masses (a four-component mixture)
#   Figure 4.11  the Zea mays differences as 15 point masses
#   Figure 4.14  the bootstrap distribution of the median of the Zea mays differences
#   Figure 4.16  a normal variance mixture with exponential variances (Laplace)
#   Figure 4.17  the asymmetric Laplace distribution AL(theta, mu, sigma)
#   Figure 4.20  samples from gamma(2, 1/3) and gamma(10, 3/2)
#   Figure 4.21  a gamma-Poisson (negative binomial) sample
#   Figure 4.22  the rootogram of a negative binomial fit (vcd::goodfit)
#   Figure 4.23  the hierarchical model gamma -> Poisson -> gamma-Poisson
#   Figure 4.24  Poisson samples for lambda = 10, ..., 100, raw and square-root scale
#   Figure 4.25  gamma-Poisson samples for mu = 1, 2, 4, ..., 1024: mean and 95% range
#   Figure 4.26  a piecewise linear variance-stabilising function built from Figure 4.25
#   Figure 4.27  the variance-stabilising transformation of Equation 4.22 for several alpha
#   Figure 4.28  the data of Exercise 4.1 (Myst.rds)
#   Figure 4.29  the NPreg data of the flexmix package
#   Figure 4.30  the NPreg data coloured by the fitted two-component regression mixture
#
# Skipped (not computed from data):
#   Figure 4.12  the sampling distribution of a statistic (schematic drawing)
#   Figure 4.13  the bootstrap principle (schematic drawing)
#   Figure 4.15  portrait of Laplace (engraving)
#   Figure 4.18  two histograms (yeast promoter lengths, microarray log-ratios) shown as images in the
#                book; the underlying data are not distributed with the book or its data folder
#   Figure 4.19  M. C. Escher artwork
#
# Two figures are drawn as the book's text describes them, which differs from the book's code:
#   Figure 4.23  the text asks for a gamma latent distribution with mean 50 and variance 30, and the
#                bottom panel is the resulting gamma-Poisson; we use shape 50^2/30 (the book's code uses
#                shape 80) and negative binomial size 50^2/30 (the book's code uses 50^2/80, which gives
#                variance 130 instead of 80).
#   Figure 4.24  ten values of lambda (the book's caption says eight).
#
# It also writes the small JSON files used by the browser demonstrations:
#   data/ch04-myst.json      the 1,800 values of Exercise 4.1, the Figure 4.5 mixture, the EM references
#   data/ch04-chipseq.json   the frequency table of ChIP-Seq tag counts (Figure 4.8) and the fitted models
#   data/ch04-zeamays.json   the 15 Zea mays differences and exact bootstrap quantities
#   data/ch04-vst.json       exact means and standard deviations of transformed Poisson and gamma-Poisson
#
# Run: Rscript ch04_mixture_models.R             (all figures and data)
#      Rscript ch04_mixture_models.R fig_4_14    (one figure)
# The code is my own; it follows the analysis steps of the book's Chapter 4.

.this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
source(file.path(.this_dir, "common.R"))

CH <- 4
CACHE <- file.path(CODE_DIR, "cache")
dir.create(CACHE, showWarnings = FALSE)

# =============================================================================================
# helpers

# ==== two-component normal mixture ===========================================================
dmix2 <- function(x, lambda, mu, sd) lambda * dnorm(x, mu[1], sd[1]) + (1 - lambda) * dnorm(x, mu[2], sd[2])
loglik_mix2 <- function(x, lambda, mu, sd) sum(log(dmix2(x, lambda, mu, sd)))

# EM for a two-component normal mixture, started from parameter values.
# One iteration = E step at the current parameters, then M step. loglik[t] is the log-likelihood
# of the parameters before iteration t (so loglik[1] is that of the start). Stops when the change
# of the log-likelihood is at most tol (after at least miniter iterations) or after maxiter.
em_mix2 <- function(x, lambda, mu, sd, tol = 1e-12, miniter = 1, maxiter = 5000, equal_sd = FALSE) {
  ll <- numeric(0)
  path <- list()
  for (it in seq_len(maxiter)) {
    pA <- lambda * dnorm(x, mu[1], sd[1])
    pB <- (1 - lambda) * dnorm(x, mu[2], sd[2])
    ll[it] <- sum(log(pA + pB))
    path[[it]] <- c(lambda = lambda, muA = mu[1], muB = mu[2], sdA = sd[1], sdB = sd[2])
    if (it == maxiter || (it > miniter && abs(ll[it] - ll[it - 1]) <= tol)) break
    wA <- pA / (pA + pB); wB <- 1 - wA                       # E step: membership weights (4.5)
    lambda <- mean(wA)                                       # M step: weighted MLEs
    mu <- c(sum(wA * x) / sum(wA), sum(wB * x) / sum(wB))
    vA <- sum(wA * (x - mu[1])^2) / sum(wA); vB <- sum(wB * (x - mu[2])^2) / sum(wB)
    sd <- if (equal_sd) rep(sqrt((sum(wA * (x - mu[1])^2) + sum(wB * (x - mu[2])^2)) / length(x)), 2)
          else sqrt(c(vA, vB))
  }
  list(lambda = lambda, mu = mu, sd = sd, loglik = ll, iter = length(ll) - 1,
       path = do.call(rbind, path), w = lambda * dnorm(x, mu[1], sd[1]) / dmix2(x, lambda, mu, sd))
}

# the deterministic start used in the browser demo: equal weights, means at the quartiles,
# both standard deviations equal to the overall (1/n) standard deviation
quartile_start <- function(x) {
  q <- unname(quantile(x, c(0.25, 0.75), type = 7))
  s <- sqrt(mean((x - mean(x))^2))
  list(lambda = 0.5, mu = q, sd = c(s, s))
}

load_myst <- function() readRDS(msmb_data("Myst.rds"))$yvar

# ==== ChIP-Seq bins (mosaicsExample data, as constructed in the book's code) ===================
load_chipseq <- function() {
  f <- file.path(CACHE, "ch04-chipseq-bins.rds")
  if (file.exists(f)) return(readRDS(f))
  suppressPackageStartupMessages(library(mosaics))
  files <- c(msmb_data("wgEncodeSydhTfbsGm12878Stat1StdAlnRep1_chr22_sorted.bam_fragL200_bin200.txt"),
             msmb_data("wgEncodeSydhTfbsGm12878InputStdAlnRep1_chr22_sorted.bam_fragL200_bin200.txt"))
  b <- suppressMessages(readBins(type = c("chip", "input"), fileName = files))
  d <- utils::capture.output(bins <- print(b))
  bins <- bins[, c("coord", "tagCount", "input")]
  saveRDS(bins, f)
  bins
}

# frequency table of counts: k = 0..max, n_k
count_table <- function(x) {
  x <- as.integer(x)
  data.frame(k = 0:max(x), n = tabulate(x + 1L, nbins = max(x) + 1L))
}

# zero-inflated Poisson by EM on a frequency table (k, n). Latent Z = 1 for a structural zero.
zip_em <- function(k, n, pi0 = 0.5, lambda = NULL, tol = 1e-12, maxiter = 10000) {
  N <- sum(n); S <- sum(k * n); n0 <- n[k == 0]
  if (is.null(lambda)) lambda <- S / N
  ll <- numeric(0)
  for (it in seq_len(maxiter)) {
    p0 <- pi0 + (1 - pi0) * exp(-lambda)
    ll[it] <- n0 * log(p0) + sum((n * (log(1 - pi0) + dpois(k, lambda, log = TRUE)))[k > 0])
    if (it > 1 && abs(ll[it] - ll[it - 1]) <= tol) break
    tau <- pi0 / p0                      # E step: P(structural zero | X = 0)
    pi0 <- n0 * tau / N                  # M step
    lambda <- S / (N - n0 * tau)
  }
  list(pi0 = pi0, lambda = lambda, loglik = ll, iter = length(ll) - 1)
}
zip_loglik <- function(k, n, pi0, lambda) {
  p <- ifelse(k == 0, pi0 + (1 - pi0) * exp(-lambda), (1 - pi0) * dpois(k, lambda))
  sum(n * log(p))
}

# negative binomial (mean mu, size a) log-likelihood on a frequency table, and its MLE.
# For fixed size the MLE of mu is the (weighted) mean, so we maximise the profile over log(size).
nb_loglik <- function(k, n, mu, size) sum(n * dnbinom(k, size = size, mu = mu, log = TRUE))
nb_mle <- function(k, n, w = rep(1, length(k))) {
  mu <- sum(w * k * n) / sum(w * n)
  f <- function(ls) -sum(w * n * dnbinom(k, size = exp(ls), mu = mu, log = TRUE))
  o <- optimize(f, c(-8, 12), tol = 1e-10)
  list(mu = mu, size = exp(o$minimum), loglik = -o$objective)
}
# zero-inflated negative binomial by EM: the M step for (mu, size) is a weighted NB MLE
zinb_em <- function(k, n, pi0 = 0.3, tol = 1e-10, maxiter = 5000) {
  N <- sum(n); n0 <- n[k == 0]
  fit <- nb_mle(k, n); mu <- fit$mu; size <- fit$size
  ll <- numeric(0)
  for (it in seq_len(maxiter)) {
    f0 <- dnbinom(0, size = size, mu = mu)
    p0 <- pi0 + (1 - pi0) * f0
    ll[it] <- n0 * log(p0) + sum((n * (log(1 - pi0) + dnbinom(k, size = size, mu = mu, log = TRUE)))[k > 0])
    if (it > 1 && abs(ll[it] - ll[it - 1]) <= tol) break
    tau <- pi0 / p0
    pi0 <- n0 * tau / N
    w <- ifelse(k == 0, 1 - tau, 1)
    fit <- nb_mle(k, n, w); mu <- fit$mu; size <- fit$size
  }
  list(pi0 = pi0, mu = mu, size = size, loglik = ll, iter = length(ll) - 1)
}

# ==== bootstrap ==============================================================================
load_zeamays <- function() HistData::ZeaMays$diff

# exact bootstrap distribution of the sample median for odd n and distinct values:
# the median of a resample is x_(m), m = (n+1)/2, and
# P(median* <= x_(j)) = P(at least m of the n draws are <= x_(j)) = P(Bin(n, j/n) >= m)
boot_median_exact <- function(x) {
  n <- length(x); stopifnot(n %% 2 == 1, !anyDuplicated(x))
  m <- (n + 1) / 2; xs <- sort(x)
  cdf <- pbinom(m - 1, n, (1:n) / n, lower.tail = FALSE)
  p <- diff(c(0, cdf))
  list(values = xs, prob = p, cdf = cdf, mean = sum(p * xs), sd = sqrt(sum(p * xs^2) - sum(p * xs)^2))
}
# percentile interval from a discrete distribution (values sorted, cdf): the type-1 quantiles
discrete_quantile <- function(values, cdf, q) sapply(q, function(qq) values[which(cdf >= qq - 1e-12)[1]])

# ==== infinite mixtures ======================================================================
# asymmetric Laplace AL(theta, mu, sigma): X | W ~ N(theta + mu W, sigma^2 W), W ~ Exp(1)
dal <- function(x, theta, mu, sigma) {
  s <- sqrt(2 * sigma^2 + mu^2); d <- x - theta
  exp((mu * d - abs(d) * s) / sigma^2) / s
}
# the variance-stabilising transformation for v(mu) = mu + alpha mu^2 (Equation 4.22), three forms.
# (x and alpha are recycled to a common length, so a scalar alpha works; ifelse() alone would not do that.)
.vst <- function(x, a, f) { n <- max(length(x), length(a)); x <- rep_len(x, n); a <- rep_len(a, n); out <- sqrt(x); i <- a > 0; out[i] <- f(x[i], a[i]); out }
g_vst_log <- function(x, a) .vst(x, a, function(x, a) log(2 * sqrt(a) * sqrt(x * (a * x + 1)) + 2 * a * x + 1) / (2 * sqrt(a)))
g_vst_acosh <- function(x, a) .vst(x, a, function(x, a) acosh(2 * a * x + 1) / (2 * sqrt(a)))
g_vst_asinh <- function(x, a) .vst(x, a, function(x, a) asinh(sqrt(a * x)) / sqrt(a))

# exact mean and sd of g(X) for a count distribution with pmf p on 0..K
moments_of <- function(g, p, k) { v <- g(k); m <- sum(p * v); c(mean = m, sd = sqrt(max(0, sum(p * v^2) - m^2))) }

# =============================================================================================
# Figures

purple <- COL[["purple"]]

fig_4_1 <- function() {
  set.seed(401)
  coin <- runif(10000) > 0.5
  x <- rnorm(length(coin), mean = ifelse(coin, 1, 3), sd = 0.5)
  p <- ggplot(data.frame(x = x), aes(x)) + geom_histogram(fill = purple, binwidth = 0.1, boundary = 0) +
    labs(x = "value", y = "count")
  savefig(p, "fig-4-1", ch = CH, width = 5, height = 3.6)
}

sim_fair <- function(n, seed) {
  set.seed(seed)
  coin <- runif(n) > 0.5
  data.frame(coin = coin, x = rnorm(n, mean = ifelse(coin, 1, 3), sd = 0.5))
}

fig_4_2 <- function() {
  d <- sim_fair(1e6, 402)
  p <- ggplot(d, aes(x)) + geom_histogram(fill = purple, bins = 200) + labs(x = "value", y = "count")
  savefig(p, "fig-4-2", ch = CH, width = 5, height = 3.6)
}

fig_4_3 <- function() {
  d <- subset(sim_fair(1e6, 402), coin)
  p <- ggplot(d, aes(x)) +
    geom_histogram(aes(y = after_stat(density)), fill = purple, binwidth = 0.01, boundary = 0) +
    stat_function(fun = dnorm, args = list(mean = 1, sd = 0.5), colour = COL[["red"]], linewidth = 0.9, n = 501) +
    labs(x = "value", y = "density")
  savefig(p, "fig-4-3", ch = CH, width = 5, height = 3.6)
}

fig_4_4 <- function() {
  d <- data.frame(x = seq(-1, 5, length.out = 1000))
  d$f <- dmix2(d$x, 0.5, c(1, 3), c(0.5, 0.5))
  p <- ggplot(d, aes(x, f)) + geom_line(colour = COL[["red"]], linewidth = 1.3) + labs(x = "x", y = "mixture density")
  savefig(p, "fig-4-4", ch = CH, width = 5, height = 3.6)
}

sim_mystery <- function() {
  set.seed(405)
  coin <- runif(1000) > 0.5
  data.frame(coin = coin, x = rnorm(1000, mean = ifelse(coin, 1, 2), sd = sqrt(0.5)))
}
mystery_breaks <- function(d) seq(min(d$x), max(d$x), length.out = 30)

fig_4_5 <- function() {
  d <- sim_mystery()
  p <- ggplot(d, aes(x)) + geom_histogram(fill = purple, breaks = mystery_breaks(d)) + labs(x = "value", y = "count")
  savefig(p, "fig-4-5", ch = CH, width = 5, height = 3.6)
}

fig_4_6 <- function() {
  d <- sim_mystery(); br <- mystery_breaks(d)
  p <- ggplot(d, aes(x)) +
    geom_histogram(data = subset(d, coin), fill = COL[["red"]], alpha = 0.35, breaks = br) +
    geom_histogram(data = subset(d, !coin), fill = COL[["blue"]], alpha = 0.35, breaks = br) +
    labs(x = "value", y = "count")
  savefig(p, "fig-4-6", ch = CH, width = 5, height = 3.6)
}

fig_4_7 <- function() {
  d <- sim_mystery(); br <- mystery_breaks(d)
  d$component <- factor(ifelse(d$coin, "heads: N(1, 0.5)", "tails: N(2, 0.5)"))
  p <- ggplot(d, aes(x, fill = component)) + geom_histogram(breaks = br, alpha = 0.75, position = "stack") +
    scale_fill_manual(values = c(COL[["red"]], COL[["blue"]])) +
    labs(x = "value", y = "count", fill = NULL) + theme(legend.position = "top")
  savefig(p, "fig-4-7", ch = CH, width = 5, height = 3.8)
}

fig_4_8 <- function() {
  b <- load_chipseq()
  p <- ggplot(b, aes(tagCount)) + geom_histogram(binwidth = 1, fill = COL[["green"]]) +
    labs(x = "tag count per 200 nt bin", y = "number of bins")
  savefig(p, "fig-4-8", ch = CH, width = 5, height = 3.6)
}

fig_4_9 <- function() {
  ct <- count_table(load_chipseq()$tagCount)
  ct <- subset(ct, n > 0)
  # bars drawn from 0.5 so that counts of one bin stay visible on the log scale
  p <- ggplot(ct) + geom_rect(aes(xmin = k - 0.5, xmax = k + 0.5, ymin = 0.5, ymax = n), fill = COL[["green"]]) +
    scale_y_log10() + labs(x = "tag count per 200 nt bin", y = "number of bins (log scale)")
  savefig(p, "fig-4-9", ch = CH, width = 5, height = 3.6)
}

fig_4_10 <- function() {
  set.seed(410)
  masses <- c(A = 331, C = 307, G = 347, T = 322)
  probs <- c(A = 0.12, C = 0.38, G = 0.36, T = 0.14)
  nuc <- sample(4, 7000, replace = TRUE, prob = probs)
  w <- rnorm(7000, mean = masses[nuc], sd = 3)
  p <- ggplot(data.frame(w = w), aes(w)) + geom_histogram(bins = 100, fill = purple) +
    labs(x = "measured mass (Da)", y = "count")
  savefig(p, "fig-4-10", ch = CH, width = 5.6, height = 3.6)
}

fig_4_11 <- function() {
  d <- data.frame(x = load_zeamays())
  p <- ggplot(d, aes(x = x, ymin = 0, ymax = 1 / 15)) + geom_linerange(linewidth = 1, colour = COL[["green"]]) +
    coord_cartesian(ylim = c(0, 0.1)) + labs(x = "height difference, crossed minus self-fertilised (inches)", y = "mass")
  savefig(p, "fig-4-11", ch = CH, width = 5.6, height = 2.6)
}

fig_4_14 <- function() {
  set.seed(414)
  x <- load_zeamays()
  meds <- replicate(1000, median(x[sample(15, 15, replace = TRUE)]))
  p <- ggplot(data.frame(m = meds), aes(m)) + geom_histogram(bins = 30, fill = purple) +
    labs(x = "bootstrap median", y = "count")
  savefig(p, "fig-4-14", ch = CH, width = 5.6, height = 3.2)
}

fig_4_16 <- function() {
  set.seed(416)
  w <- rexp(10000, rate = 1)
  x <- rnorm(10000, mean = 0.3, sd = sqrt(w))
  b <- 1 / sqrt(2)
  p <- ggplot(data.frame(x = x), aes(x)) + geom_histogram(fill = purple, binwidth = 0.1, boundary = 0) +
    stat_function(fun = function(t) 10000 * 0.1 * exp(-abs(t - 0.3) / b) / (2 * b), colour = COL[["orange"]],
                  linewidth = 0.8, n = 801) +
    labs(x = "x", y = "count")
  savefig(p, "fig-4-16", ch = CH, width = 5, height = 3.6)
}

fig_4_17 <- function() {
  set.seed(417)
  mu <- 0.3; sigma <- 0.4; theta <- -1
  w <- rexp(10000, 1)
  x <- rnorm(10000, theta + mu * w, sigma * sqrt(w))
  p <- ggplot(data.frame(x = x), aes(x)) + geom_histogram(fill = purple, binwidth = 0.1, boundary = 0) +
    stat_function(fun = function(t) 10000 * 0.1 * dal(t, theta, mu, sigma), colour = COL[["orange"]],
                  linewidth = 0.8, n = 801) +
    labs(x = "x", y = "count")
  savefig(p, "fig-4-17", ch = CH, width = 5, height = 3.6)
}

fig_4_20 <- function() {
  set.seed(420)
  a <- ggplot(data.frame(x = rgamma(10000, shape = 2, rate = 1 / 3)), aes(x)) +
    geom_histogram(bins = 100, fill = purple) + labs(title = "(a) gamma(shape 2, rate 1/3)", x = "x", y = "count")
  b <- ggplot(data.frame(x = rgamma(10000, shape = 10, rate = 3 / 2)), aes(x)) +
    geom_histogram(bins = 100, fill = purple) + labs(title = "(b) gamma(shape 10, rate 3/2)", x = "x", y = "count")
  p <- patchwork::wrap_plots(a, b, ncol = 2) & theme(plot.title = element_text(size = 10))
  savefig(p, "fig-4-20", ch = CH, width = 7.4, height = 3.2)
}

sim_gp <- function() {
  set.seed(421)
  lambda <- rgamma(10000, shape = 10, rate = 3 / 2)
  rpois(10000, lambda = lambda)
}

fig_4_21 <- function() {
  gp <- sim_gp()
  p <- ggplot(data.frame(x = gp), aes(x)) + geom_histogram(binwidth = 1, fill = purple) + labs(x = "count", y = "frequency")
  savefig(p, "fig-4-21", ch = CH, width = 5, height = 3.4)
}

fig_4_22 <- function() {
  suppressPackageStartupMessages(library(vcd))
  gp <- sim_gp()
  ofit <- goodfit(gp, "nbinomial")
  savefig(function() plot(ofit, xlab = "count", main = ""), "fig-4-22", ch = CH, width = 7.4, height = 3.8)
  invisible(ofit)
}

fig_4_23 <- function() {
  x <- 0:95; mu <- 50; vtot <- 80; v1 <- vtot - mu
  shape <- mu^2 / v1; scale <- v1 / mu
  p1 <- dgamma(x, shape = shape, scale = scale)
  p2 <- dpois(x, lambda = 60)
  p3 <- dnbinom(x, mu = mu, size = shape)
  pal <- RColorBrewer::brewer.pal(8, "Paired")
  draw <- function() {
    op <- par(mfrow = c(3, 1), mai = c(0.42, 0.85, 0.12, 0.08), mgp = c(3.1, 0.7, 0))
    on.exit(par(op))
    plot(NA, xlim = range(x), ylim = c(0, 0.08), xlab = "", ylab = "gamma density", las = 1)
    polygon(c(x[1], x, x[length(x)]), c(0, p1, 0), col = pal[1], border = pal[2])
    abline(v = mu, lwd = 3); abline(v = 60, col = pal[2], lty = 2, lwd = 3)
    plot(x, p2, type = "h", lwd = 2, col = pal[3], xlab = "", ylab = "Poisson(60)", las = 1)
    abline(v = 60, col = pal[2], lwd = 2); abline(v = 55, col = pal[4], lty = 2, lwd = 3)
    plot(x, p3, type = "h", lwd = 2, col = pal[4], xlab = "count", ylab = "gamma-Poisson", las = 1)
  }
  savefig(draw, "fig-4-23", ch = CH, width = 6.4, height = 6)
}

fig_4_24 <- function() {
  set.seed(424)
  lam <- seq(10, 100, by = 10)
  d <- do.call(rbind, lapply(lam, function(l) { n <- rpois(200, l); data.frame(lambda = l, n = n) }))
  long <- rbind(data.frame(lambda = d$lambda, name = "n", value = d$n),
                data.frame(lambda = d$lambda, name = "sqrt(n)", value = sqrt(d$n)))
  p <- ggplot(long, aes(factor(lambda), value)) + geom_violin(fill = "grey90") +
    facet_grid(rows = vars(name), scales = "free") + labs(x = expression(lambda), y = NULL)
  savefig(p, "fig-4-24", ch = CH, width = 5.2, height = 5.4)
  invisible(d)
}

sim_nb_series <- function() {
  set.seed(425)
  muv <- 2^(0:10)
  do.call(rbind, lapply(muv, function(mu) {
    u <- rnbinom(1e4, mu = mu, size = 4)
    data.frame(mu = mu, mean = mean(u), sd = sd(u), lower = unname(quantile(u, 0.025)), upper = unname(quantile(u, 0.975)))
  }))
}

fig_4_25 <- function() {
  s <- sim_nb_series()
  p <- ggplot(s, aes(mu, mean, ymin = lower, ymax = upper)) + geom_point() + geom_errorbar(width = 20) +
    labs(x = expression(mu), y = "sample mean and 95% range")
  savefig(p, "fig-4-25", ch = CH, width = 5, height = 3.6)
}

fig_4_26 <- function() {
  s <- sim_nb_series()
  s$book <- cumsum(s$mean / s$sd)                       # the book's construction
  s$riemann <- cumsum(c(s$mean[1], diff(s$mean)) / s$sd) # integral of 1/sd(mu) d mu
  long <- rbind(data.frame(mean = s$mean, value = s$book, how = "cumsum(mean / sd)"),
                data.frame(mean = s$mean, value = 2 * s$riemann, how = "2 x cumsum(diff(mean) / sd)"))
  p <- ggplot(long, aes(mean, value, colour = how, shape = how)) + geom_line() + geom_point(size = 2) +
    scale_colour_manual(values = c(COL[["orange"]], "black")) + labs(x = "mean", y = "transformed value", colour = NULL, shape = NULL) +
    theme(legend.position = "top")
  savefig(p, "fig-4-26", ch = CH, width = 5, height = 3.9)
}

fig_4_27 <- function() {
  x <- seq(0, 24, by = 0.1)
  d <- do.call(rbind, lapply(c(0, 0.05 * 2^(0:5)), function(a) data.frame(x = x, a = a, y = g_vst_log(x, a))))
  p <- ggplot(d, aes(x, y, colour = factor(a))) + geom_line(linewidth = 0.8) +
    scale_colour_viridis_d(end = 0.9) + labs(x = "x", y = expression(g[alpha](x)), colour = expression(alpha))
  savefig(p, "fig-4-27", ch = CH, width = 5.6, height = 3.8)
}

fig_4_28 <- function() {
  mx <- load_myst()
  fit <- em_mix2(mx, quartile_start(mx)$lambda, quartile_start(mx)$mu, quartile_start(mx)$sd)
  xs <- seq(min(mx), max(mx), length.out = 400)
  dens <- data.frame(x = xs, f = 1800 * 0.025 * dmix2(xs, fit$lambda, fit$mu, fit$sd))
  p <- ggplot(data.frame(mx = mx), aes(mx)) + geom_histogram(binwidth = 0.025, fill = "grey45", boundary = 0) +
    geom_line(data = dens, aes(x, f), colour = COL[["orange"]], linewidth = 0.8) + labs(x = "mx", y = "count")
  savefig(p, "fig-4-28", ch = CH, width = 5, height = 3.6)
}

fit_npreg <- function() {
  suppressPackageStartupMessages(library(flexmix))
  data("NPreg", package = "flexmix", envir = environment())
  set.seed(429)
  m1 <- flexmix(yn ~ x + I(x^2), data = NPreg, k = 2)
  list(data = NPreg, model = m1)
}

fig_4_29 <- function() {
  data("NPreg", package = "flexmix", envir = environment())
  p <- ggplot(NPreg, aes(x, yn)) + geom_point(size = 1.4) + labs(x = "x", y = "yn")
  savefig(p, "fig-4-29", ch = CH, width = 4.4, height = 4)
}

fig_4_30 <- function() {
  f <- fit_npreg()
  d <- f$data; d$cluster <- factor(modeltools::clusters(f$model))
  p <- ggplot(d, aes(x, yn, colour = cluster, shape = cluster)) + geom_point(size = 1.6) +
    scale_colour_manual(values = c(COL[["blue"]], COL[["orange"]])) + labs(x = "x", y = "yn", colour = "fitted\ncomponent", shape = "fitted\ncomponent")
  savefig(p, "fig-4-30", ch = CH, width = 5, height = 4)
}

# =============================================================================================
# data for the browser demonstrations

data_ch04 <- function() {
  # Exercise 4.1 data, Figure 4.5 mixture, Question 4.7 sample, with EM references from the demo's start
  mx <- load_myst()
  st <- quartile_start(mx)
  fit <- em_mix2(mx, st$lambda, st$mu, st$sd)
  my <- sim_mystery()
  stm <- quartile_start(my$x)
  fitm <- em_mix2(my$x, stm$lambda, stm$mu, stm$sd)
  set.seed(407)
  y7 <- c(rnorm(100, mean = -0.2, sd = 0.5), rnorm(50, mean = 0.5, sd = 1))
  st7 <- quartile_start(y7)
  fit7 <- em_mix2(y7, st7$lambda, st7$mu, st7$sd)
  ref <- function(f) list(lambda = f$lambda, mu = f$mu, sd = f$sd, loglik = tail(f$loglik, 1), iter = f$iter,
                          loglik_first = head(f$loglik, 6))
  export_json(list(source = "Holmes and Huber (2019), data file Myst.rds (Exercise 4.1); Figure 4.5 and Question 4.7 samples simulated by ch04_mixture_models.R",
                   myst = mx, mystery = my$x, mystery_label = as.integer(!my$coin) + 1L, q47 = y7,
                   ref = list(myst = ref(fit), mystery = ref(fitm), q47 = ref(fit7)),
                   note = "EM from the quartile start, tolerance 1e-12 on the log-likelihood change"),
              "ch04-myst.json", digits = 15)  # full precision, so that the demo's EM can match R closely

  # ChIP-Seq frequency table and fitted models
  ct <- count_table(load_chipseq()$tagCount)
  zp <- zip_em(ct$k, ct$n)
  nb <- nb_mle(ct$k, ct$n)
  zn <- zinb_em(ct$k, ct$n)
  export_json(list(source = "mosaicsExample (Kuan et al. 2011): STAT1 ChIP-Seq, GM12878, chromosome 22, 200 nt bins (Holmes and Huber 2019, Section 4.2.3)",
                   k = ct$k, n = ct$n,
                   ref = list(poisson = list(lambda = sum(ct$k * ct$n) / sum(ct$n)),
                              zip = list(pi0 = zp$pi0, lambda = zp$lambda, loglik = tail(zp$loglik, 1), iter = zp$iter),
                              nb = list(mu = nb$mu, size = nb$size, loglik = nb$loglik),
                              zinb = list(pi0 = zn$pi0, mu = zn$mu, size = zn$size, loglik = tail(zn$loglik, 1)))),
              "ch04-chipseq.json", digits = 12)

  # Zea mays and exact bootstrap quantities
  x <- load_zeamays()
  ex <- boot_median_exact(x)
  n <- length(x)
  export_json(list(source = "HistData::ZeaMays (Darwin's data), differences crossed minus self-fertilised, inches",
                   diff = x,
                   ref = list(median = median(x), mean = mean(x),
                              boot_median_values = ex$values, boot_median_prob = ex$prob,
                              boot_median_sd = ex$sd, boot_median_mean = ex$mean,
                              boot_mean_sd = sqrt(mean((x - mean(x))^2) / n),
                              ci99_median = discrete_quantile(ex$values, ex$cdf, c(0.005, 0.995)),
                              ci95_median = discrete_quantile(ex$values, ex$cdf, c(0.025, 0.975)),
                              n_resamples = format(choose(2 * n - 1, n - 1), scientific = FALSE))), "ch04-zeamays.json", digits = 12)

  # exact moments of transformed counts
  K <- 0:20000
  lam <- seq(10, 100, by = 10)
  pois <- t(sapply(lam, function(l) { p <- dpois(K, l); c(lambda = l, moments_of(identity, p, K), sqrt_sd = moments_of(sqrt, p, K)[["sd"]]) }))
  muv <- 2^(0:10); a <- 1 / 4
  nbm <- t(sapply(muv, function(m) {
    p <- dnbinom(K, mu = m, size = 4)
    c(mu = m, sd_raw = moments_of(identity, p, K)[["sd"]], sd_sqrt = moments_of(sqrt, p, K)[["sd"]],
      sd_log1 = moments_of(function(v) log(v + 1), p, K)[["sd"]], sd_vst = moments_of(function(v) g_vst_asinh(v, a), p, K)[["sd"]])
  }))
  export_json(list(note = "exact standard deviations by summing the probability mass function over 0..20000",
                   poisson = as.data.frame(pois), nb_size4 = as.data.frame(nbm)), "ch04-vst.json", digits = 12)
}

# Set .ch04_no_run before sourcing this file to reuse the helpers without plotting
if (!exists(".ch04_no_run")) {
  run_figures(list(fig_4_1 = fig_4_1, fig_4_2 = fig_4_2, fig_4_3 = fig_4_3, fig_4_4 = fig_4_4, fig_4_5 = fig_4_5,
                   fig_4_6 = fig_4_6, fig_4_7 = fig_4_7, fig_4_8 = fig_4_8, fig_4_9 = fig_4_9, fig_4_10 = fig_4_10,
                   fig_4_11 = fig_4_11, fig_4_14 = fig_4_14, fig_4_16 = fig_4_16, fig_4_17 = fig_4_17,
                   fig_4_20 = fig_4_20, fig_4_21 = fig_4_21, fig_4_22 = fig_4_22, fig_4_23 = fig_4_23,
                   fig_4_24 = fig_4_24, fig_4_25 = fig_4_25, fig_4_26 = fig_4_26, fig_4_27 = fig_4_27,
                   fig_4_28 = fig_4_28, fig_4_29 = fig_4_29, fig_4_30 = fig_4_30, data_ch04 = data_ch04))
}
