# Chapter 2. Statistical Modeling
# Course: Statistical Methods for Modern Biology (following Holmes and Huber 2019,
# Modern Statistics for Modern Biology, Cambridge University Press, Chapter 2).
#
# Reproduces every figure of the chapter that is computed from data or by simulation:
#   Figure 2.2   bar chart of the epitope counts without the outlier (e99)
#   Figure 2.3   hanging rootogram of e99 against the fitted Poisson
#   Figure 2.4   Poisson log-likelihood of e100 as a function of lambda
#   Figure 2.5   binomial likelihood of the colour-blindness data (10 of 120)
#   Figure 2.6   binomial log-likelihood for y = 40 successes in n = 300 trials
#   Figure 2.7   null distribution of the chi-square statistic for ten S. aureus genes (B = 1000)
#   Figure 2.8   QQ plot of that simulated statistic against chi-square with 30 degrees of freedom
#   Figure 2.9   Chargaff's nucleotide percentages, one bar chart per organism
#   Figure 2.10  null distribution of Chargaff's statistic under per-row permutation (1e5 permutations)
#   Figure 2.11  Hardy-Weinberg log-likelihood for the Tahiti sample of the Mourant data
#   Figure 2.12  de Finetti (ternary) diagram of five Mourant populations with the HWE curve
#   Figure 2.13  sequence logo of the Kozak motif
#   Figure 2.14  four-state Markov chain fitted to a short DNA sequence (transition probabilities)
#   Figure 2.19  three Beta priors with the same mean 1/4
#   Figure 2.20  marginal (beta-binomial) distribution of Y, simulated
#   Figure 2.21  posterior of p given Y = 40 by conditioning the simulation, with Beta(90, 610)
#   Figure 2.22  QQ plot of the two posterior samples
#   Figure 2.23  likelihoods, prior and posteriors of a log fold change for two genes (recomputed on
#                the pasilla data with DESeq2; the book shows a published figure of Love et al. 2014)
#   Figure 2.24  Poissonness plot for Shine-Dalgarno motif counts in 50 kb windows of E. coli
#   Figure 2.25  exponential plot of the gaps between Shine-Dalgarno motifs
#   Figure 2.26  CpG islands in chr8:2.2-5.8 Mb (a genome-axis track, without the ideogram)
#   Figure 2.27  Markov-chain log-likelihood-ratio scores of island and non-island 100-mers (hg19 chr8)
#   Figure 2.28  GC content in sliding windows of width 100 along S. aureus gene 364 (Exercise 2.4)
#   Figure 2.29  the same with a lowess smooth (Exercise 2.4)
#   Figure 2.30  five Beta densities (Exercise 2.5)
#
# Skipped (not computed from data):
#   Figure 2.1   diagram of a probability model generating data (schematic)
#   Figure 2.15  "turtles all the way down" drawing of hierarchical models (cartoon)
#   Figure 2.16  definition of a short tandem repeat (schematic)
#   Figure 2.17  positions of STRs on the Y chromosome (schematic taken from a web database)
#   Figure 2.18  Y-STR haplotype lookup in a database (screenshot)
# Figure 2.26 omits the chromosome ideogram, which Gviz downloads from UCSC at run time.
#
# It also writes the JSON files used by the browser demonstrations:
#   data/ch02-counts.json    epitope counts, Shine-Dalgarno window counts, Mourant genotypes
#   data/ch02-multinom.json  nucleotide counts of the first 60 S. aureus genes, Chargaff's table
#   data/ch02-cpg.json       chr8 CpG-island and background Markov chains, scores, sample windows
#
# Run: Rscript ch02_statistical_modeling.R            (all figures and data)
#      Rscript ch02_statistical_modeling.R fig_2_7    (one figure)
# Figure 2.27 and data/ch02-cpg.json need BSgenome.Hsapiens.UCSC.hg19 on the first run; the
# transition counts and sampled windows are then cached in cache/ch02-cpg.rds.
# The code is my own; it follows the analysis steps of the book's Chapter 2.

.this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
source(file.path(.this_dir, "common.R"))
suppressPackageStartupMessages({
  library(Biostrings)
})

CH <- 2
CACHE <- file.path(CODE_DIR, "cache")
dir.create(CACHE, showWarnings = FALSE)

# ---------------------------------------------------------------------------------------------
# helpers

load_rdata <- function(file, name) {
  env <- new.env(); load(msmb_data(file), envir = env); env[[name]]
}
load_e100 <- function() as.integer(load_rdata("e100.RData", "e100"))
load_chargaff <- function() load_rdata("ChargaffTable.RData", "ChargaffTable")
load_deuteranopia <- function() load_rdata("Deuteranopia.RData", "Deuteranopia")
load_kozak <- function() load_rdata("kozak.RData", "kozak")
load_staph <- function() readDNAStringSet(msmb_data("staphsequence.ffn.txt"), "fasta")
load_mourant <- function() {
  env <- new.env(); utils::data("Mourant", package = "HardyWeinberg", envir = env); env$Mourant
}
colourblind <- function() c(rep(0L, 110), rep(1L, 10))

# Poisson log-likelihood of a vector of counts
pois_loglik <- function(lambda, x) vapply(lambda, function(l) sum(dpois(x, l, log = TRUE)), 0)

# likelihood-ratio (G^2) and Pearson goodness-of-fit statistics for a fitted Poisson, cells 0..max(x);
# df follows the convention of vcd::goodfit (observed non-empty cells minus 1 minus 1 estimated parameter)
pois_gof <- function(x) {
  lam <- mean(x); n <- length(x); k <- 0:max(x)
  obs <- tabulate(x + 1L, nbins = max(x) + 1L); fit <- n * dpois(k, lam)
  G2 <- 2 * sum(ifelse(obs > 0, obs * log(obs / fit), 0))
  X2 <- sum((obs - fit)^2 / fit)
  df <- sum(obs > 0) - 2
  list(lambda = lam, k = k, obs = obs, fit = fit, G2 = G2, X2 = X2, df = df,
       p = pchisq(G2, df, lower.tail = FALSE))
}

# nucleotide counts (rows A, C, G, T) of the staph genes
staph_counts <- function() {
  f <- file.path(CACHE, "ch02-staph-counts.rds")
  if (file.exists(f)) return(readRDS(f))
  staph <- load_staph()
  m <- vapply(staph, letterFrequency, FUN.VALUE = numeric(4), letters = "ACGT", OR = 0)
  colnames(m) <- paste0("gene", seq_along(staph)); saveRDS(m, f); m
}
pearson <- function(obs, expd) sum((obs - expd)^2 / expd)

# Chargaff's statistic, and its exact permutation distribution: a uniformly random permutation of a
# row only matters through the pairing of its four values into {A,T} and {C,G}; there are three
# pairings, each with probability 8/24 = 1/3, and rows are permuted independently.
stat_chf <- function(x) sum((x[, "C"] - x[, "G"])^2 + (x[, "A"] - x[, "T"])^2)
chf_row_values <- function(v) {
  c((v[1] - v[2])^2 + (v[3] - v[4])^2, (v[1] - v[3])^2 + (v[2] - v[4])^2, (v[1] - v[4])^2 + (v[2] - v[3])^2)
}
chf_exact_p <- function(tab) {
  vals <- apply(tab[, c("A", "T", "C", "G")], 1, chf_row_values)  # 3 x 8; first entry is the observed pairing
  tot <- 0
  for (j in seq_len(ncol(vals))) tot <- outer(tot, vals[, j], "+")
  tot <- as.vector(tot)
  obs <- stat_chf(tab)
  list(obs = obs, p = mean(tot <= obs + 1e-9), n = length(tot), dist = tot)
}

# Hardy-Weinberg: MLE of the allele frequency by gene counting, expected counts and Pearson statistic
hwe_fit <- function(nMM, nMN, nNN) {
  S <- nMM + nMN + nNN; p <- (2 * nMM + nMN) / (2 * S); q <- 1 - p
  e <- S * c(p^2, 2 * p * q, q^2); o <- c(nMM, nMN, nNN)
  X2 <- sum((o - e)^2 / e)
  list(p = p, S = S, expected = e, X2 = X2, pval = pchisq(X2, 1, lower.tail = FALSE),
       se = sqrt(p * q / (2 * S)))
}
hwe_loglik <- function(p, nMM, nMN, nNN) 2 * nMM * log(p) + nMN * log(2 * p * (1 - p)) + 2 * nNN * log(1 - p)

# first-order Markov chain: transition counts and maximum-likelihood transition matrix
transition_counts <- function(s, letters = c("A", "C", "G", "T")) {
  x <- factor(s[-length(s)], letters); y <- factor(s[-1], letters)
  unclass(table(x, y))
}
transition_mle <- function(counts) counts / rowSums(counts)

# Beta-binomial (marginal of Y when p ~ Beta(a, b) and Y | p ~ Bin(n, p))
dbetabinom <- function(y, n, a, b, log = FALSE) {
  l <- lchoose(n, y) + lbeta(a + y, b + n - y) - lbeta(a, b)
  if (log) l else exp(l)
}
# highest posterior density interval of a unimodal Beta(a, b), a, b > 1
hpd_beta <- function(a, b, level = 0.95) {
  f <- function(lo) qbeta(pbeta(lo, a, b) + level, a, b) - lo
  lo <- optimize(f, c(0, qbeta(1 - level, a, b)), tol = 1e-12)$minimum
  c(lo, qbeta(pbeta(lo, a, b) + level, a, b))
}

# CpG data: transition counts in islands and outside, and sampled windows (cached; needs hg19 once)
cpg_cache_file <- file.path(CACHE, "ch02-cpg.rds")
cpg_data <- function() {
  if (file.exists(cpg_cache_file)) return(readRDS(cpg_cache_file))
  if (!requireNamespace("BSgenome.Hsapiens.UCSC.hg19", quietly = TRUE))
    stop("BSgenome.Hsapiens.UCSC.hg19 is needed once to build ", cpg_cache_file, call. = FALSE)
  suppressPackageStartupMessages({ library(BSgenome.Hsapiens.UCSC.hg19); library(IRanges) })
  tab <- read.table(msmb_data("model-based-cpg-islands-hg19.txt"), header = TRUE)
  t8 <- tab[tab$chr == "chr8", ]
  ir <- IRanges(start = t8$start, end = t8$end)
  chr8 <- unmasked(BSgenome.Hsapiens.UCSC.hg19::Hsapiens$chr8)
  isl <- as(Views(chr8, ir), "DNAStringSet")
  non <- as(Views(chr8, gaps(ir)), "DNAStringSet")
  di <- function(s) matrix(colSums(dinucleotideFrequency(s)), 4, byrow = TRUE,
                           dimnames = list(DNA_BASES, DNA_BASES))
  base <- function(s) colSums(letterFrequency(s, DNA_BASES))
  # sampled windows: drop sequences with letters other than ACGT, pick sequences with probability
  # proportional to (length - len), then a uniform start (my implementation of the book's scheme)
  windows <- function(s, len, B, seed) {
    set.seed(seed)
    af <- alphabetFrequency(s)
    s <- s[rowSums(af[, setdiff(colnames(af), DNA_BASES), drop = FALSE]) == 0]
    w <- pmax(width(s) - len, 0)
    idx <- sample(length(s), B, replace = TRUE, prob = w / sum(w))
    st <- vapply(idx, function(i) sample.int(width(s)[i] - len, 1), 0L)
    as.character(subseq(s[idx], start = st + 1L, width = len))
  }
  out <- list(TI = di(isl), TN = di(non), fI = base(isl), fN = base(non),
              n_isl = length(isl), n_non = length(non),
              w100_isl = windows(isl, 100, 1000, 21), w100_non = windows(non, 100, 1000, 22),
              w300_isl = windows(isl, 300, 300, 23), w300_non = windows(non, 300, 300, 24))
  saveRDS(out, cpg_cache_file)
  out
}
cpg_model <- function(cd = cpg_data()) {
  MI <- cd$TI / rowSums(cd$TI); MN <- cd$TN / rowSums(cd$TN)
  pI <- cd$fI / sum(cd$fI); pN <- cd$fN / sum(cd$fN)
  list(MI = MI, MN = MN, pI = pI, pN = pN, alpha = log(pI / pN), beta = log(MI / MN))
}
# log-likelihood-ratio score of a sequence (character string), total or per base
cpg_score <- function(x, M, per_base = FALSE) {
  s <- strsplit(x, "")[[1]]
  sc <- M$alpha[[s[1]]]
  if (length(s) > 1) sc <- sc + sum(M$beta[cbind(s[-length(s)], s[-1])])
  if (per_base) sc / length(s) else sc
}

# ---------------------------------------------------------------------------------------------
# Section 2.3: the epitope data

fig_2_2 <- function() {
  e99 <- load_e100(); e99 <- e99[-which.max(e99)]
  d <- as.data.frame(table(k = e99))
  p <- ggplot(d, aes(k, Freq)) + geom_col(width = 0.55, fill = COL[["green"]]) +
    labs(x = "positive calls at a position", y = "number of positions")
  savefig(p, "fig-2-2", ch = CH, width = 4.2, height = 3.2)
}

# hanging rootogram: a bar of height sqrt(observed) hangs from sqrt(expected)
rootogram_plot <- function(x, title = NULL) {
  g <- pois_gof(x); k <- 0:(max(x) + 1)
  obs <- tabulate(x + 1L, nbins = length(k)); fit <- length(x) * dpois(k, g$lambda)
  d <- data.frame(k = k, top = sqrt(fit), bottom = sqrt(fit) - sqrt(obs), fit = sqrt(fit))
  ggplot(d) +
    geom_rect(aes(xmin = k - 0.4, xmax = k + 0.4, ymin = bottom, ymax = top), fill = COL[["green"]], alpha = 0.8) +
    geom_hline(yintercept = 0, colour = "grey30") +
    geom_line(aes(k, fit), colour = COL[["red"]]) + geom_point(aes(k, fit), colour = COL[["red"]], size = 2) +
    scale_x_continuous(breaks = k) +
    labs(x = "count k", y = "sqrt(frequency)", title = title)
}
fig_2_3 <- function() {
  e99 <- load_e100(); e99 <- e99[-which.max(e99)]
  p <- rootogram_plot(e99, sprintf("e99 against Poisson(%.2f)", mean(e99)))
  savefig(p, "fig-2-3", ch = CH, width = 4.4, height = 3.4)
}

fig_2_4 <- function() {
  e100 <- load_e100()
  lam <- seq(0.05, 0.95, length.out = 100)
  d <- data.frame(lambda = lam, ll = pois_loglik(lam, e100))
  m0 <- mean(e100)
  p <- ggplot(d, aes(lambda, ll)) + geom_line(colour = COL[["red"]], linewidth = 1) +
    geom_vline(xintercept = m0, colour = COL[["blue"]], linewidth = 0.8) +
    geom_hline(yintercept = pois_loglik(m0, e100), colour = COL[["purple"]], linewidth = 0.8) +
    labs(x = expression(lambda), y = "log-likelihood")
  savefig(p, "fig-2-4", ch = CH, width = 4.6, height = 3.4)
}

# Section 2.4: binomial likelihood
fig_2_5 <- function() {
  cb <- colourblind()
  probs <- seq(0, 0.3, by = 0.005)
  d <- data.frame(p = probs, L = dbinom(sum(cb), size = length(cb), prob = probs))
  p <- ggplot(d, aes(p, L)) + geom_point(size = 1.2) +
    geom_vline(xintercept = 1 / 12, colour = COL[["blue"]], linetype = 2) +
    labs(x = "probability of success p", y = "likelihood")
  savefig(p, "fig-2-5", ch = CH, width = 4.4, height = 3.2)
}
fig_2_6 <- function() {
  p_seq <- seq(0.001, 0.999, by = 0.001)
  d <- data.frame(p = p_seq, ll = dbinom(40, 300, p_seq, log = TRUE))
  p <- ggplot(d, aes(p, ll)) + geom_line() + labs(x = "p", y = "log f(p | y)")
  savefig(p, "fig-2-6", ch = CH, width = 4, height = 3.4)
}

# Sections 2.5 and 2.6: multinomial counts of ten genes
staph_null <- function(B = 1000, seed = 2) {
  tab10 <- staph_counts()[, 1:10]
  p0 <- rowMeans(apply(tab10, 2, function(x) x / sum(x)))
  cs <- colSums(tab10); E <- outer(p0, cs)
  set.seed(seed)
  sim <- replicate(B, pearson(sapply(cs, function(s) rmultinom(1, s, p0)), E))
  list(tab10 = tab10, p0 = p0, cs = cs, E = E, S1 = pearson(tab10, E), sim = sim)
}
fig_2_7 <- function() {
  r <- staph_null()
  q <- quantile(r$sim, c(0.95, 0.99))
  p <- ggplot(data.frame(s = r$sim), aes(s)) +
    geom_histogram(breaks = seq(0, 75, length.out = 50), fill = "#d9d2ec", colour = "grey40", linewidth = 0.2) +
    geom_vline(xintercept = r$S1, colour = COL[["red"]], linewidth = 0.9) +
    geom_vline(xintercept = q, colour = c(COL[["green"]], COL[["blue"]]), linetype = 2) +
    labs(x = "simulated statistic", y = "count")
  savefig(p, "fig-2-7", ch = CH, width = 4.6, height = 3.4)
}
fig_2_8 <- function() {
  r <- staph_null()
  d <- data.frame(th = qchisq(ppoints(length(r$sim)), df = 30), s = sort(r$sim))
  p <- ggplot(d, aes(th, s)) + geom_abline(colour = COL[["red"]]) + geom_point(size = 0.7) +
    coord_equal() + labs(x = expression(chi[nu == 30]^2 ~ "quantiles"), y = "simulated statistic")
  savefig(p, "fig-2-8", ch = CH, width = 4, height = 4.2)
}

# Section 2.7: Chargaff
fig_2_9 <- function() {
  ct <- load_chargaff()
  d <- data.frame(org = factor(rep(rownames(ct), 4), rownames(ct)),
                  base = factor(rep(colnames(ct), each = nrow(ct)), rev(colnames(ct))),
                  pct = as.vector(ct))
  pal <- c("#8c510a", "#35978f", "#2166ac", "#d6604d", "#4d9221", "#b8860b", "#008b8b", "#8b1a1a")
  p <- ggplot(d, aes(pct, base, fill = org)) + geom_col(width = 0.7, show.legend = FALSE) +
    facet_wrap(~org, nrow = 2) + scale_fill_manual(values = pal) +
    scale_x_continuous(breaks = c(0, 10, 20, 30)) +
    labs(x = "percentage of the mass", y = NULL)
  savefig(p, "fig-2-9", ch = CH, width = 8, height = 4)
}
chargaff_perm <- function(B = 1e5, seed = 3) {
  f <- file.path(CACHE, "ch02-chargaff-perm.rds")
  if (file.exists(f)) return(readRDS(f))
  ct <- load_chargaff(); set.seed(seed)
  perm <- replicate(B, { x <- t(apply(ct, 1, sample)); colnames(x) <- colnames(ct); stat_chf(x) })
  saveRDS(perm, f); perm
}
fig_2_10 <- function() {
  ct <- load_chargaff(); perm <- chargaff_perm(); obs <- stat_chf(ct)
  p <- ggplot(data.frame(s = perm), aes(s)) +
    geom_histogram(bins = 100, fill = "#d9d2ec", colour = "grey40", linewidth = 0.1) +
    geom_vline(xintercept = obs, colour = COL[["red"]], linewidth = 0.9) +
    labs(x = "Chargaff statistic under permutation", y = "count")
  savefig(p, "fig-2-10", ch = CH, width = 4.6, height = 3.4)
}

# Hardy-Weinberg
fig_2_11 <- function() {
  M <- load_mourant(); r <- M[216, ]
  xv <- seq(0.01, 0.99, by = 0.01)
  d <- data.frame(p = xv, ll = hwe_loglik(xv, r$MM, r$MN, r$NN))
  ph <- hwe_fit(r$MM, r$MN, r$NN)$p
  p <- ggplot(d, aes(p, ll)) + geom_line(linewidth = 1) +
    geom_vline(xintercept = ph, colour = COL[["blue"]]) +
    geom_hline(yintercept = hwe_loglik(ph, r$MM, r$MN, r$NN), colour = COL[["purple"]]) +
    labs(x = "p", y = "log-likelihood")
  savefig(p, "fig-2-11", ch = CH, width = 4.4, height = 3.4)
}
fig_2_12 <- function() {
  suppressPackageStartupMessages(library(HardyWeinberg))
  M <- load_mourant(); pops <- c(1, 69, 128, 148, 192)
  gf <- as.matrix(M[, c("MM", "MN", "NN")])
  draw <- function() {
    par(mai = rep(0.1, 4))
    invisible(capture.output(HWTernaryPlot(gf[pops, ], markerlab = M$Country[pops], alpha = 1e-4,
                                           curvecols = c("red", rep("purple", 4)), mcex = 0.75, vertex.cex = 1)))
  }
  savefig(draw, "fig-2-12", ch = CH, width = 4.6, height = 4.4)
}

# Kozak logo
fig_2_13 <- function() {
  suppressPackageStartupMessages(library(seqLogo))
  pwm <- makePWM(load_kozak())
  savefig(function() seqLogo(pwm, ic.scale = FALSE), "fig-2-13", ch = CH, width = 5, height = 4)
}

# Section 2.8: a four-state chain estimated from a short sequence
mc_sequence <- function() {
  strsplit("ACACGTTTTCCACGTACCCAAATACGGCATGTGTGAGCTG", "")[[1]]
}
fig_2_14 <- function() {
  P <- transition_mle(transition_counts(mc_sequence()))
  xy <- rbind(A = c(-1, 1), C = c(1, 1), G = c(1, -1), T = c(-1, -1)); r <- 0.2
  draw <- function() {
    par(mai = c(0, 0, 0, 0))
    plot(NA, xlim = c(-1.6, 1.6), ylim = c(-1.6, 1.6), asp = 1, axes = FALSE, xlab = "", ylab = "")
    for (a in rownames(P)) for (b in colnames(P)) {
      if (P[a, b] == 0) next
      if (a == b) {  # self-transition: a loop on the outer side of the node
        u <- xy[a, ] / sqrt(sum(xy[a, ]^2)); cc <- xy[a, ] + u * (r + 0.13)
        th <- seq(0, 2 * pi, length.out = 60)
        lines(cc[1] + 0.14 * cos(th), cc[2] + 0.14 * sin(th), col = "grey45")
        text(cc[1] + u[1] * 0.27, cc[2] + u[2] * 0.27, sprintf("%.2f", P[a, b]), col = "#1f3f7f", cex = 0.8)
        next
      }
      p0 <- xy[a, ]; p1 <- xy[b, ]; d <- p1 - p0; L <- sqrt(sum(d^2)); u <- d / L; nrm <- c(-u[2], u[1])
      off <- 0.09 * nrm  # the two directions of a pair are drawn side by side
      s0 <- p0 + u * r + off; s1 <- p1 - u * r + off
      arrows(s0[1], s0[2], s1[1], s1[2], length = 0.08, col = "grey45")
      lp <- p0 + d * 0.62 + 2.3 * off
      text(lp[1], lp[2], sprintf("%.2f", P[a, b]), col = "#1f3f7f", cex = 0.75)
    }
    th <- seq(0, 2 * pi, length.out = 80)
    for (a in rownames(xy)) {
      polygon(xy[a, 1] + r * cos(th), xy[a, 2] + r * sin(th), col = "#f2c18d", border = "black")
      text(xy[a, 1], xy[a, 2], a, cex = 1.3, font = 2)
    }
  }
  savefig(draw, "fig-2-14", ch = CH, width = 4.6, height = 4.6)
}

# Section 2.9: Bayesian thinking
beta_curves <- function(pars, labels) {
  p_seq <- seq(0, 1, by = 0.001)
  do.call(rbind, lapply(seq_along(labels), function(i)
    data.frame(p = p_seq, dens = dbeta(p_seq, pars[[i]][1], pars[[i]][2]), pars = labels[i])))
}
fig_2_19 <- function() {
  lab <- c("Beta(10,30)", "Beta(20,60)", "Beta(50,150)")
  d <- beta_curves(list(c(10, 30), c(20, 60), c(50, 150)), lab); d$pars <- factor(d$pars, lab)
  p <- ggplot(d, aes(p, dens, colour = pars)) + geom_line(linewidth = 0.9) +
    scale_colour_manual(values = unname(COL[c("blue", "orange", "green")]), name = NULL) +
    coord_cartesian(xlim = c(0, 0.6)) + labs(x = "p", y = "density")
  savefig(p, "fig-2-19", ch = CH, width = 5, height = 3.2)
}
bayes_sim <- function(seed = 4) {
  set.seed(seed)
  rp <- rbeta(1e5, 50, 350)
  y <- rbinom(length(rp), size = 300, prob = rp)
  list(rp = rp, y = y, post = rp[y == 40])
}
fig_2_20 <- function() {
  s <- bayes_sim()
  p <- ggplot(data.frame(y = s$y), aes(y)) + geom_bar(width = 0.9, fill = COL[["orange"]]) +
    labs(x = "y", y = "count")
  savefig(p, "fig-2-20", ch = CH, width = 4.6, height = 3.2)
}
fig_2_21 <- function() {
  s <- bayes_sim(); p_seq <- seq(0, 1, by = 0.001)
  p <- ggplot(data.frame(p = s$post), aes(p)) +
    geom_histogram(aes(y = after_stat(density)), breaks = seq(0.08, 0.18, by = 0.0025), fill = COL[["green"]],
                   colour = "white", linewidth = 0.1) +
    geom_line(data = data.frame(p = p_seq, d = dbeta(p_seq, 90, 610)), aes(p, d), linewidth = 1) +
    coord_cartesian(xlim = c(0.08, 0.18)) + labs(x = "posterior p", y = "density")
  savefig(p, "fig-2-21", ch = CH, width = 4.6, height = 3.4)
}
fig_2_22 <- function() {
  s <- bayes_sim(); set.seed(5)
  mc <- rbeta(1e5, 90, 610)
  pr <- ppoints(length(s$post))
  d <- data.frame(x = quantile(mc, pr, names = FALSE), y = sort(s$post))
  p <- ggplot(d, aes(x, y)) + geom_abline(colour = COL[["blue"]]) + geom_line() + coord_equal() +
    labs(x = "quantiles of the Beta(90, 610) sample", y = "quantiles of the conditioned sample")
  savefig(p, "fig-2-22", ch = CH, width = 4.2, height = 4.2)
}

# Figure 2.23 recomputed: NB likelihood of the log2 fold change of two pasilla genes, with the
# normal prior that DESeq2 estimates (lfcShrink(type = "normal")) and the resulting posteriors
lfc_example <- function() {
  f <- file.path(CACHE, "ch02-lfc.rds")
  if (file.exists(f)) return(readRDS(f))
  suppressPackageStartupMessages(library(DESeq2))
  cnt <- as.matrix(read.csv(system.file("extdata", "pasilla_gene_counts.tsv", package = "pasilla"),
                            sep = "\t", row.names = "gene_id"))
  ann <- read.csv(system.file("extdata", "pasilla_sample_annotation.csv", package = "pasilla"), row.names = 1)
  rownames(ann) <- sub("fb$", "", rownames(ann))
  ann <- ann[colnames(cnt), ]; ann$condition <- factor(ann$condition, c("untreated", "treated"))
  dds <- DESeqDataSetFromMatrix(cnt, ann, ~condition)
  dds <- dds[rowSums(counts(dds)) >= 10, ]
  dds <- suppressMessages(DESeq(dds, quiet = TRUE))
  res <- results(dds)
  shr <- suppressMessages(lfcShrink(dds, coef = 2, type = "normal", quiet = TRUE))
  pv <- priorInfo(shr)$betaPriorVar[2]
  disp <- dispersions(dds); sf <- sizeFactors(dds); x <- as.integer(ann$condition == "treated")
  ok <- which(res$baseMean > 100 & abs(res$log2FoldChange - 1) < 0.15 & !is.na(res$padj))
  g1 <- ok[which.min(disp[ok])]; g2 <- ok[which.max(disp[ok])]
  grid <- seq(-1, 3, length.out = 801)
  curve_for <- function(i) {
    k <- counts(dds)[i, ]; a <- disp[i]
    ll <- vapply(grid, function(b1) {
      nll <- function(b0) -sum(dnbinom(k, size = 1 / a, mu = sf * 2^(b0 + b1 * x), log = TRUE))
      -optimize(nll, c(-5, 25))$objective
    }, 0)
    h <- grid[2] - grid[1]
    lik <- exp(ll - max(ll)); lik <- lik / sum(lik * h)
    post <- lik * dnorm(grid, 0, sqrt(pv)); post <- post / sum(post * h)
    j <- which.max(post)
    lp <- log(post); se <- sqrt(-1 / ((lp[j + 1] - 2 * lp[j] + lp[j - 1]) / h^2))
    data.frame(lfc = grid, lik = lik, post = post, gene = rownames(dds)[i], map = grid[j], se = se,
               mle = res$log2FoldChange[i], disp = a, shr = shr$log2FoldChange[i])
  }
  out <- list(d = rbind(curve_for(g1), curve_for(g2)), prior_var = pv)
  saveRDS(out, f); out
}
fig_2_23 <- function() {
  L <- lfc_example(); d <- L$d
  d$gene <- factor(d$gene, unique(d$gene))
  lab <- setNames(sprintf("%s (dispersion %.3f)", levels(d$gene), tapply(d$disp, d$gene, `[`, 1)), levels(d$gene))
  pr <- data.frame(lfc = unique(d$lfc)); pr$dens <- dnorm(pr$lfc, 0, sqrt(L$prior_var))
  mp <- unique(d[, c("gene", "map", "se")]); mp$y <- c(-0.15, -0.3) * max(d$post)
  cols <- setNames(c(COL[["green"]], COL[["purple"]]), levels(d$gene))
  p <- ggplot(d, aes(lfc, colour = gene)) +
    geom_line(data = pr, aes(lfc, dens), colour = "black", inherit.aes = FALSE, linewidth = 0.8) +
    geom_line(aes(y = lik), linewidth = 0.9) + geom_line(aes(y = post), linetype = 2, linewidth = 0.9) +
    geom_errorbar(data = mp, aes(y = y, xmin = map - se, xmax = map + se), width = 0.1, orientation = "y",
                  inherit.aes = FALSE, colour = cols[as.character(mp$gene)]) +
    geom_point(data = mp, aes(map, y), inherit.aes = FALSE, colour = cols[as.character(mp$gene)]) +
    scale_colour_manual(values = cols, labels = lab, name = NULL) +
    labs(x = expression(log[2] ~ "fold change"), y = "density") + theme(legend.position = "bottom")
  savefig(p, "fig-2-23", ch = CH, width = 5.6, height = 4)
}

# Section 2.10: Shine-Dalgarno motif in E. coli
sd_data <- function() {
  f <- file.path(CACHE, "ch02-shine-dalgarno.rds")
  if (file.exists(f)) return(readRDS(f))
  suppressPackageStartupMessages(library(BSgenome.Ecoli.NCBI.20080805))
  ecoli <- BSgenome.Ecoli.NCBI.20080805::Ecoli$NC_010473
  w <- 50000; st <- seq(1, length(ecoli) - w, by = w)
  nm <- vapply(st, function(s) countPattern("AGGAGGT", ecoli[s:(s + w - 1)], max.mismatch = 0), 0)
  m <- matchPattern("AGGAGGT", ecoli, max.mismatch = 0)
  out <- list(numMatches = as.integer(nm), gaps = width(gaps(m)), starts = start(m), genome_length = length(ecoli))
  saveRDS(out, f); out
}
# Poissonness plot: metameter log(k! n_k / N) against k with delta-method 95% intervals
poissonness <- function(x) {
  N <- length(x); k <- 0:max(x); nk <- tabulate(x + 1L, nbins = length(k))
  ok <- nk > 0; k <- k[ok]; nk <- nk[ok]
  phi <- lfactorial(k) + log(nk / N); se <- sqrt((1 - nk / N) / nk)
  data.frame(k = k, nk = nk, phi = phi, lo = phi - 1.96 * se, hi = phi + 1.96 * se)
}
fig_2_24 <- function() {
  x <- sd_data()$numMatches; d <- poissonness(x); lam <- mean(x)
  fitln <- lm(phi ~ k, weights = nk, data = d)
  p <- ggplot(d, aes(k, phi)) + geom_errorbar(aes(ymin = lo, ymax = hi), width = 0.12, colour = "grey50") +
    geom_abline(intercept = -lam, slope = log(lam), colour = COL[["red"]], linewidth = 0.8) +
    geom_abline(intercept = coef(fitln)[1], slope = coef(fitln)[2], colour = COL[["blue"]], linetype = 2) +
    geom_point(size = 2.2) +
    labs(x = "number of occurrences k", y = "log(k! n_k / N)",
         subtitle = sprintf("red: Poisson(%.3f) line; blue dashed: weighted LS slope %.3f", lam, coef(fitln)[2]))
  savefig(p, "fig-2-24", ch = CH, width = 4.8, height = 3.8)
}
fig_2_25 <- function() {
  g <- sd_data()$gaps; n <- length(g); rate <- 1 / mean(g)
  d <- data.frame(x = sort(g), q = -log(1 - ppoints(n)))
  p <- ggplot(d, aes(x, q)) + geom_point(size = 1.5) +
    geom_abline(intercept = 0, slope = rate, colour = "black") +
    labs(x = "gap between motifs (bp)", y = "exponential quantile -log(1 - F)",
         subtitle = sprintf("line: fitted rate 1/mean = %.3g per bp", rate))
  savefig(p, "fig-2-25", ch = CH, width = 4.8, height = 3.6)
}

# Section 2.10.1: CpG islands
fig_2_26 <- function() {
  tab <- read.table(msmb_data("model-based-cpg-islands-hg19.txt"), header = TRUE)
  t8 <- tab[tab$chr == "chr8" & tab$end >= 2200000 & tab$start <= 5800000, ]
  p <- ggplot(t8) + geom_rect(aes(xmin = start / 1e6, xmax = pmax(end, start + 2500) / 1e6, ymin = 0, ymax = 1),
                              fill = "#006400") +
    scale_x_continuous(limits = c(2.2, 5.8), breaks = seq(2.5, 5.5, 0.5)) +
    scale_y_continuous(breaks = NULL) +
    labs(x = "chr8 position (Mb, hg19)", y = "CpG", subtitle = sprintf("%d model-based CpG islands", nrow(t8))) +
    theme(panel.grid = element_blank())
  savefig(p, "fig-2-26", ch = CH, width = 6.4, height = 1.8)
}
fig_2_27 <- function() {
  cd <- cpg_data(); M <- cpg_model(cd)
  sI <- vapply(cd$w100_isl, cpg_score, 0, M = M, per_base = TRUE)
  sN <- vapply(cd$w100_non, cpg_score, 0, M = M, per_base = TRUE)
  br <- seq(min(sI, sN), max(sI, sN), length.out = 50)
  d <- rbind(data.frame(s = sI, type = "CpG island"), data.frame(s = sN, type = "elsewhere"))
  p <- ggplot(d, aes(s, fill = type)) + geom_histogram(breaks = br, position = "identity", alpha = 0.45, colour = NA) +
    scale_fill_manual(values = c("CpG island" = "#0000ff", elsewhere = "#ff0000"), name = NULL) +
    coord_cartesian(xlim = c(-0.5, 0.5)) + labs(x = "log-likelihood-ratio score per base", y = "count")
  savefig(p, "fig-2-27", ch = CH, width = 5, height = 3.4)
}

# Exercises 2.4 and 2.5
gc_window <- function(i = 364, window = 100) {
  s <- load_staph()[[i]]
  rowSums(letterFrequencyInSlidingView(s, window, c("G", "C"))) / window
}
fig_2_28 <- function() {
  gc <- gc_window()
  p <- ggplot(data.frame(i = seq_along(gc), gc = gc), aes(i, gc)) + geom_line(linewidth = 0.3) +
    labs(x = "window start", y = "GC fraction")
  savefig(p, "fig-2-28", ch = CH, width = 5.6, height = 3)
}
fig_2_29 <- function() {
  gc <- gc_window(); lw <- lowess(seq_along(gc), gc, f = 0.2)
  p <- ggplot(data.frame(i = seq_along(gc), gc = gc), aes(i, gc)) + geom_line(linewidth = 0.3) +
    geom_line(data = data.frame(i = lw$x, gc = lw$y), colour = COL[["red"]], linewidth = 1) +
    labs(x = "window start", y = "GC fraction")
  savefig(p, "fig-2-29", ch = CH, width = 5.6, height = 3)
}
fig_2_30 <- function() {
  lab <- c("Beta(0.5,0.5)", "U(0,1)=Beta(1,1)", "Beta(10,30)", "Beta(20,60)", "Beta(50,150)")
  d <- beta_curves(list(c(.5, .5), c(1, 1), c(10, 30), c(20, 60), c(50, 150)), lab)
  d$pars <- factor(d$pars, lab); d <- d[is.finite(d$dens), ]
  p <- ggplot(d, aes(p, dens, colour = pars)) + geom_line(linewidth = 0.8) +
    geom_vline(xintercept = 0.25, colour = "#990000", linetype = 2) +
    scale_colour_manual(values = unname(COL[c("purple", "grey", "blue", "orange", "green")]), name = NULL) +
    coord_cartesian(ylim = c(0, 13)) + labs(x = "p", y = "density")
  savefig(p, "fig-2-30", ch = CH, width = 5.4, height = 3.4)
}

# ---------------------------------------------------------------------------------------------
# data for the browser demonstrations
data_ch02 <- function() {
  e100 <- load_e100(); sd <- sd_data(); M <- load_mourant()
  export_json(list(source = "Holmes and Huber (2019), Chapter 2: e100.RData; E. coli NC_010473 (BSgenome.Ecoli.NCBI.20080805); HardyWeinberg::Mourant",
                   e100 = e100, shine_dalgarno = sd$numMatches, sd_gaps = sd$gaps,
                   mourant = list(population = M$Population, country = M$Country,
                                  MM = M$MM, MN = M$MN, NN = M$NN)), "ch02-counts.json")
  lf <- staph_counts()[, 1:60]
  ct <- load_chargaff()
  export_json(list(source = "S. aureus genes (staphsequence.ffn.txt) and ChargaffTable.RData, Holmes and Huber (2019), Chapter 2",
                   letters = c("A", "C", "G", "T"), staph = unname(lapply(seq_len(ncol(lf)), function(j) as.integer(lf[, j]))),
                   chargaff = list(organism = rownames(ct), A = ct[, "A"], T = ct[, "T"], C = ct[, "C"], G = ct[, "G"])),
              "ch02-multinom.json")
  if (file.exists(cpg_cache_file) || requireNamespace("BSgenome.Hsapiens.UCSC.hg19", quietly = TRUE)) {
    cd <- cpg_data()
    export_json(list(source = "hg19 chr8 (BSgenome.Hsapiens.UCSC.hg19) with the model-based CpG islands of Irizarry et al. (2009)",
                     letters = DNA_BASES, TI = unname(cd$TI), TN = unname(cd$TN),
                     fI = unname(cd$fI), fN = unname(cd$fN),
                     isl = cd$w300_isl, non = cd$w300_non), "ch02-cpg.json", digits = 8)
  } else message("skipping data/ch02-cpg.json: BSgenome.Hsapiens.UCSC.hg19 not installed yet")
}

# Set .ch02_no_run before sourcing this file to reuse the helpers without plotting
if (!exists(".ch02_no_run")) {
  run_figures(list(fig_2_2 = fig_2_2, fig_2_3 = fig_2_3, fig_2_4 = fig_2_4, fig_2_5 = fig_2_5,
                   fig_2_6 = fig_2_6, fig_2_7 = fig_2_7, fig_2_8 = fig_2_8, fig_2_9 = fig_2_9,
                   fig_2_10 = fig_2_10, fig_2_11 = fig_2_11, fig_2_12 = fig_2_12, fig_2_13 = fig_2_13,
                   fig_2_14 = fig_2_14, fig_2_19 = fig_2_19, fig_2_20 = fig_2_20, fig_2_21 = fig_2_21,
                   fig_2_22 = fig_2_22, fig_2_23 = fig_2_23, fig_2_24 = fig_2_24, fig_2_25 = fig_2_25,
                   fig_2_26 = fig_2_26, fig_2_27 = fig_2_27, fig_2_28 = fig_2_28, fig_2_29 = fig_2_29,
                   fig_2_30 = fig_2_30, data_ch02 = data_ch02))
}
