# Chapter 1. Generative Models for Discrete Data
# Course: Statistical Methods for Modern Biology (following Holmes and Huber 2019,
# Modern Statistics for Modern Biology, Cambridge University Press, Chapter 1).
#
# Reproduces every figure of the chapter that is computed from data or by simulation:
#   Figure 1.1   Poisson(5) probabilities of 0, ..., 12 mutations (HIV example)
#   Figure 1.3   the binomial B(15, 0.3) probability mass function
#   Figure 1.5   300,000 simulated binomial counts B(10000, 5e-4) (the mutation process)
#   Figure 1.7   typical background data: 100 independent Poisson(0.5) counts
#   Figure 1.8   the observed ELISA epitope counts e100 (50 patients, 100 positions)
#   Figure 1.10  the simulated null distribution of the chi-square statistic, n = 20, four letters
#
# Skipped (not computed from data):
#   Figure 1.2   two boxes receiving balls with probabilities 1/3 and 2/3 (hand-drawn schematic)
#   Figure 1.4   portrait of Simeon Poisson (photograph / engraving)
#   Figure 1.6   diagram of an antibody (schematic)
#   Figure 1.9   four boxes of unequal sizes (hand-drawn schematic)
#   Figure 1.11  diagram of a probability model F(theta) generating data x (schematic)
#
# It also writes the small JSON files used by the browser demonstrations:
#   data/ch01-e100.json       the 100 epitope counts
#   data/ch01-celegans.json   nucleotide counts of the C. elegans mitochondrial genome (Exercise 1.8)
#
# Run: Rscript ch01_generative_discrete.R            (all figures and data)
#      Rscript ch01_generative_discrete.R fig_1_10   (one figure)
# The code is my own; it follows the analysis steps of the book's Chapter 1.

.this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
source(file.path(.this_dir, "common.R"))

CH <- 1

# =============================================================================================
# helpers

# the epitope data: counts of positive ELISA calls at 100 positions, summed over 50 patients
load_e100 <- function() {
  env <- new.env()
  load(msmb_data("e100.RData"), envir = env)
  as.integer(env$e100)
}

# Pearson's statistic sum (x - e)^2 / e, one value per column of a count matrix
pearson_stat <- function(counts, expected) {
  counts <- as.matrix(counts)
  colSums((counts - expected)^2 / expected)
}

# P(max of n iid Poisson(lambda) >= m), exactly and with the book's exponential approximation
pmax_pois <- function(m, n, lambda, approx = FALSE) {
  eps <- ppois(m - 1, lambda, lower.tail = FALSE)
  if (approx) -expm1(-n * eps) else -expm1(n * log1p(-eps))
}

# bar chart of a probability mass function (theoretical values in red, as in the book)
pmf_bars <- function(k, prob, fill = COL[["red"]], xlab = "k", ylab = "probability") {
  ggplot(data.frame(k = k, p = prob), aes(k, p)) +
    geom_col(fill = fill, colour = "grey25", linewidth = 0.25, width = 0.8) +
    scale_x_continuous(breaks = k) +
    scale_y_continuous(expand = expansion(mult = c(0, 0.05))) +
    labs(x = xlab, y = ylab)
}

# =============================================================================================
# Figure 1.1: Poisson(5) probabilities of 0, ..., 12 mutations (HIV, 1e4 sites x 5e-4 per site)
fig_1_1 <- function() {
  k <- 0:12
  p <- pmf_bars(k, dpois(k, lambda = 5), xlab = "number of mutations k", ylab = "P(X = k)") +
    ggtitle(expression("Poisson(" * lambda == 5 * ")"))
  savefig(p, "fig-1-1", ch = CH, width = 5.6, height = 3.6)
}

# Figure 1.3: the binomial B(15, 0.3)
fig_1_3 <- function() {
  k <- 0:15
  p <- pmf_bars(k, dbinom(k, size = 15, prob = 0.3), xlab = "number of successes k", ylab = "P(X = k)") +
    ggtitle("Binomial B(15, 0.3): mode at k = 4")
  savefig(p, "fig-1-3", ch = CH, width = 5.6, height = 3.6)
}

# Figure 1.5: 300,000 simulated mutation counts B(10000, 5e-4), with the Poisson(5) probabilities
fig_1_5 <- function() {
  set.seed(20190105)
  sims <- rbinom(300000, size = 10000, prob = 5e-4)
  tab <- as.data.frame(table(factor(sims, levels = 0:max(sims))), stringsAsFactors = FALSE)
  names(tab) <- c("k", "count")
  tab$k <- as.integer(tab$k)
  tab$pois <- 300000 * dpois(tab$k, 5)
  p <- ggplot(tab, aes(k, count)) +
    geom_col(fill = "#c8b6e2", colour = "grey30", linewidth = 0.25, width = 0.8) +
    geom_point(aes(y = pois), colour = COL[["red"]], size = 1.6) +
    scale_x_continuous(breaks = seq(0, max(tab$k), 2)) +
    scale_y_continuous(labels = function(x) format(x, big.mark = ",", scientific = FALSE),
                       expand = expansion(mult = c(0, 0.05))) +
    labs(x = "simulated number of mutations", y = "count in 300,000 simulations",
         title = "B(10000, 5e-4) simulated (bars) and 300,000 x Poisson(5) (red points)")
  savefig(p, "fig-1-5", ch = CH, width = 6.4, height = 3.8)
}

# epitope bar chart used by Figures 1.7 and 1.8
epitope_bars <- function(counts, fill, title, highlight = NULL, ymax = 7.4) {
  d <- data.frame(pos = seq_along(counts), count = counts)
  d$hi <- if (is.null(highlight)) FALSE else d$pos %in% highlight
  p <- ggplot(d, aes(pos, count)) +
    geom_col(aes(fill = hi), width = 0.7, colour = NA, show.legend = FALSE) +
    scale_fill_manual(values = c(`FALSE` = fill, `TRUE` = COL[["red"]])) +
    scale_x_continuous(breaks = c(1, seq(10, 100, 10)), limits = c(0.3, 100.7), expand = c(0, 0)) +
    scale_y_continuous(breaks = 0:7, limits = c(0, ymax), expand = c(0, 0)) +
    labs(x = "position along the protein", y = "positive calls (of 50 patients)", title = title)
  if (!is.null(highlight))
    p <- p + annotate("text", x = highlight, y = counts[highlight] + 0.25, label = "?", colour = COL[["red"]],
                      fontface = "bold", size = 5, vjust = 0)
  p
}

# Figure 1.7: typical background data, 100 independent Poisson(0.5) counts
fig_1_7 <- function() {
  set.seed(20190107)
  s100 <- rpois(100, lambda = 0.5)
  p <- epitope_bars(s100, "#c8b6e2", "Simulated background: 100 independent Poisson(0.5) counts")
  savefig(p, "fig-1-7", ch = CH, width = 6.4, height = 3.4)
}

# Figure 1.8: the observed epitope data e100 (the spike of 7 is at position 42)
fig_1_8 <- function() {
  e100 <- load_e100()
  p <- epitope_bars(e100, "darkolivegreen", "Observed ELISA counts e100: 50 patients, 100 positions",
                    highlight = which.max(e100), ymax = 8.1)
  savefig(p, "fig-1-8", ch = CH, width = 6.4, height = 3.4)
}

# Figure 1.10: the null distribution of Pearson's statistic for n = 20 letters, four equally likely
# nucleotides, from 1000 simulated sequences; the chi-square(3) density (red) is added for comparison.
fig_1_10 <- function() {
  set.seed(20190110)
  pvec <- rep(1 / 4, 4)
  obs0 <- rmultinom(1000, size = 20, prob = pvec)
  S0 <- pearson_stat(obs0, 20 * pvec)
  q95 <- quantile(S0, 0.95)
  d <- data.frame(S0 = S0)
  xx <- seq(0.01, 20, length.out = 400)
  # the statistic lives on the lattice 0, 0.4, 0.8, ...: bins of width 0.8 centred on lattice pairs
  p <- ggplot(d, aes(S0)) +
    geom_histogram(aes(y = after_stat(density)), binwidth = 0.8, boundary = -0.2,
                   fill = "#c8b6e2", colour = "grey30", linewidth = 0.25) +
    geom_line(data = data.frame(x = xx, y = dchisq(xx, 3)), aes(x, y), colour = COL[["red"]], linewidth = 0.8) +
    geom_vline(xintercept = q95, linetype = 2, colour = COL[["blue"]]) +
    annotate("text", x = q95 + 0.3, y = 0.2, hjust = 0, colour = COL[["blue"]], size = 3.4,
             label = sprintf("simulated 95%% quantile = %.1f", q95)) +
    scale_y_continuous(expand = expansion(mult = c(0, 0.05))) +
    coord_cartesian(xlim = c(0, 20)) +
    labs(x = "statistic S0 (Equation 1.1)", y = "density",
         title = "1000 fair sequences of length 20; red: chi-square density, 3 df")
  savefig(p, "fig-1-10", ch = CH, width = 6.0, height = 3.8)
}

# data for the browser demonstrations
data_ch01 <- function() {
  e100 <- load_e100()
  export_json(list(source = "Holmes and Huber (2019), Chapter 1, data file e100.RData",
                   description = "positive ELISA calls at 100 positions, summed over 50 patients",
                   counts = e100), "ch01-e100.json")
  suppressPackageStartupMessages(library(BSgenome.Celegans.UCSC.ce2))
  lf <- Biostrings::letterFrequency(BSgenome.Celegans.UCSC.ce2::Celegans$chrM, letters = c("A", "C", "G", "T"))
  export_json(list(source = "BSgenome.Celegans.UCSC.ce2 (UCSC ce2), chromosome M",
                   letters = c("A", "C", "G", "T"), counts = as.integer(lf)), "ch01-celegans.json")
}

# numerical answers quoted in the solutions of the exercises (printed, no figure)
exercises_ch01 <- function() {
  cat("Exercise 1.2: P(X = 2) =", dbinom(2, 10, 0.3), " P(X <= 2) =", pbinom(2, 10, 0.3), "\n")
  e100 <- load_e100(); l99 <- mean(e100[-which.max(e100)])
  cat("Exercise 1.3: P(max >= 7), lambda = 0.5: exact", pmax_pois(7, 100, 0.5), " approx", pmax_pois(7, 100, 0.5, TRUE),
      "\n              lambda = mean(e100) = 0.55:", pmax_pois(7, 100, mean(e100)),
      "  lambda = mean without the spike =", l99, ":", pmax_pois(7, 100, l99), "\n")
  p9 <- pmax_pois(9, 100, 0.5)
  succ <- function(B) { k <- 0:300; sum(dpois(k, B * p9)[qbeta(0.95, k + 1, B - k) < 1e-6]) }
  cat("Exercise 1.5: P(max >= 9) =", p9, "; runs needed with no hit:", ceiling(log(0.05) / log1p(-1e-6)),
      "; probability that B runs certify p < 1e-6 (95% Clopper-Pearson):",
      paste(sprintf("B = %g: %.3f", c(3e6, 1e7, 1.5e7, 2e7), sapply(c(3e6, 1e7, 1.5e7, 2e7), succ)), collapse = ", "), "\n")
  set.seed(1); x <- rpois(100, 3)
  cat("Exercise 1.7 (set.seed(1)): mean", mean(x), " var", var(x), "; theory: SD of the mean", sqrt(3 / 100),
      ", SD of the sample variance", sqrt((3 * (1 + 3 * 3) - 97 / 99 * 9) / 100), "\n")
}

# Set .ch01_no_run before sourcing this file to reuse the helpers without plotting
if (!exists(".ch01_no_run")) {
  run_figures(list(fig_1_1 = fig_1_1, fig_1_3 = fig_1_3, fig_1_5 = fig_1_5, fig_1_7 = fig_1_7,
                   fig_1_8 = fig_1_8, fig_1_10 = fig_1_10, data_ch01 = data_ch01,
                   exercises_ch01 = exercises_ch01))
}
