# Chapter 13. Design of High Throughput Experiments and their Analyses
# Course: Statistical Methods for Modern Biology, following Holmes and Huber (2019),
# Modern Statistics for Modern Biology, Chapter 13.
#
# Reproduces every figure of the chapter that is computed from data or by simulation:
#   Figure 13.2   confounding of disease state and batch (the book's own data, designI.rda)
#   Figure 13.4   effect size as the distance between the group medians
#   Figure 13.5   the same comparison with 6 and with 60 samples
#   Figure 13.7   Hotelling's weighing design: log2 ratio of the two sums of squared errors
#   Figure 13.8   the n = 6 data coloured by time of collection (blocking)
#   Figure 13.10  p-value distributions of the paired and the unpaired t-test
#   Figure 13.12  opinion poll: independent versus clustered (correlated) sampling
#
# Figures not reproduced (not computed from data):
#   Figure 13.1   photograph of a coin-tossing machine
#   Figure 13.3   portrait of Ibn Sina (Avicenna)
#   Figure 13.6   photograph of a pharmacist's balance
#   Figure 13.9   drawing of Darwin's paired maize pots (schematic)
#   Figure 13.11  photograph (the elephant in the room)
#   Figure 13.13  photograph of the 1896 Ford Quadricycle
#   Figure 13.14  drawing of a leaky pipeline (schematic)
#
# Data exported for the browser demonstrations (courses/biostat/data/):
#   ch13-darwin.json   Darwin's Zea mays heights (Fisher 1935), from HistData::ZeaMays
#
# Run:  Rscript ch13_experimental_design.R            all figures
#       Rscript ch13_experimental_design.R fig_13_7   one figure

here <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
source(file.path(here, "common.R"))

suppressPackageStartupMessages({
  library(dplyr)
  library(tidyr)
  library(ggbeeswarm)
  library(gridExtra)
})

CH <- 13

# ===================================================================================================
# The biomarker data of Section 13.4. The book's data folder ships them as designI.rda with older
# variable names (states healthy/tumor, times, batch); we rename them to match the figures.
load_design_data <- function() {
  e <- new.env()
  load(msmb_data("designI.rda"), envir = e)
  b6 <- e$batdef
  small <- tibble(
    state  = factor(ifelse(b6$states == "healthy", "healthy", "disease"), levels = c("healthy", "disease")),
    exprst = b6$exprst,                     # data with a time (block) effect
    exprs0 = b6$exprs0,                     # data with a batch effect
    time   = factor(b6$times),
    batch  = factor(sub("Batch", "Batch ", b6$batch)))
  big <- tibble(
    state = factor(ifelse(e$batN60$stateN == "healthy", "healthy", "disease"), levels = c("healthy", "disease")),
    exprs = e$batN60$exprsN)
  list(small = small, big = big)
}

# One beeswarm-plus-boxplot panel; col = NULL draws black points.
biomarker_panel <- function(d, y, col = NULL, ylim = NULL, cex = 6) {
  p <- ggplot(d, aes(x = state, y = .data[[y]])) +
    geom_boxplot(alpha = 0.5, colour = COL[["blue"]], outlier.shape = NA) +
    labs(x = "state", y = "biomarker level")
  p <- if (is.null(col)) p + geom_beeswarm(size = 2, cex = cex)
       else p + geom_beeswarm(aes(colour = .data[[col]]), size = 2, cex = cex) +
                scale_colour_manual(values = unname(COL[c("orange", "teal", "purple")]), name = col)
  if (!is.null(ylim)) p <- p + coord_cartesian(ylim = ylim)
  p
}

# Double-headed arrow between the two group medians, drawn at x = 1.5.
median_arrow <- function(d, y) {
  m <- tapply(d[[y]], d$state, median)
  annotate("segment", x = 1.5, xend = 1.5, y = m[["healthy"]], yend = m[["disease"]],
           colour = COL[["red"]], linewidth = 0.8,
           arrow = grid::arrow(length = grid::unit(0.3, "cm"), ends = "both", type = "closed"))
}

# ===================================================================================================
# Figure 13.2: the biomarker separates the two states perfectly, but state and batch coincide.
fig_13_2 <- function() {
  d <- load_design_data()$small
  p0 <- biomarker_panel(d, "exprs0")
  p1 <- biomarker_panel(d, "exprs0", col = "batch")
  g <- arrangeGrob(p0, p1, ncol = 2, widths = c(1.3, 2))
  savefig(g, "fig-13-2", ch = CH, width = 6.4, height = 3.2)
}

# Figure 13.4: the effect size as the difference between the group medians.
fig_13_4 <- function() {
  d <- load_design_data()$small
  p <- biomarker_panel(d, "exprs0", cex = 5) + median_arrow(d, "exprs0")
  savefig(p, "fig-13-4", ch = CH, width = 2.8, height = 3.2)
}

# Figure 13.5: 6 versus 60 samples per group, same error distribution.
fig_13_5 <- function() {
  D <- load_design_data()
  yl <- range(c(D$small$exprst, D$big$exprs)) + c(-0.1, 0.1)
  pa <- biomarker_panel(D$small, "exprst", ylim = yl) + ggtitle("6 per group")
  pb <- biomarker_panel(D$big, "exprs", ylim = yl, cex = 2) + median_arrow(D$big, "exprs") + ggtitle("60 per group")
  g <- arrangeGrob(pa, pb, ncol = 2, widths = c(1.6, 2.5))
  savefig(g, "fig-13-5", ch = CH, width = 6.0, height = 3.2)
}

# Figure 13.8: the 6-per-group data of Figure 13.5, coloured by the time of collection (a block factor).
fig_13_8 <- function() {
  d <- load_design_data()$small
  yl <- range(d$exprst) + c(-0.1, 0.1)
  pa <- biomarker_panel(d, "exprst", ylim = yl)
  pb <- biomarker_panel(d, "exprst", col = "time", ylim = yl)
  g <- arrangeGrob(pa, pb, ncol = 2, widths = c(1.3, 2))
  savefig(g, "fig-13-8", ch = CH, width = 6.4, height = 3.2)
}

# ===================================================================================================
# Section 13.4.3: Hotelling's weighing design. The 8 x 8 Sylvester-Hadamard matrix with entries +-1.
hadamard8 <- function() {
  H <- matrix(1, 1, 1)
  for (k in 1:3) H <- rbind(cbind(H, H), cbind(H, -H))
  H
}

# One replicate: weights theta, naive design (each weight once) and the Hadamard design.
weighing_sse <- function(C, sigma = 0.1) {
  theta <- round(2 * sample(8, 8) + rnorm(8), 1)
  x <- theta + rnorm(8, 0, sigma)                    # method 1: one object per weighing
  y <- drop(C %*% theta) + rnorm(8, 0, sigma)        # method 2: weighing j measures sum_i C[j, i] theta_i
  thetahat <- drop(crossprod(C, y)) / nrow(C)        # least squares, since C'C = 8 I
  c(naive = sum((x - theta)^2), hadamard = sum((thetahat - theta)^2))
}

fig_13_7 <- function() {
  set.seed(1307)
  C <- hadamard8()
  sse <- replicate(10000, weighing_sse(C))
  lr <- log2(sse["naive", ] / sse["hadamard", ])
  # Exact law: SSE1 / SSE2 = 8 F with F ~ F(8, 8), so log2 ratio = 3 + log2 F.
  # Density of L = 3 + log2 F: f_L(l) = f_F(2^(l-3)) * 2^(l-3) * log(2).
  xx <- seq(min(lr), max(lr), length.out = 400)
  dens <- df(2^(xx - 3), 8, 8) * 2^(xx - 3) * log(2)
  bw <- diff(range(lr)) / 50
  p <- ggplot(tibble(lr = lr), aes(x = lr)) +
    geom_histogram(bins = 50, fill = "grey55", colour = "white", linewidth = 0.2) +
    geom_line(data = tibble(x = xx, y = dens * length(lr) * bw), aes(x, y), colour = COL[["blue"]], linewidth = 0.7) +
    geom_vline(xintercept = log2(8), colour = COL[["orange"]], linewidth = 0.8) +
    labs(x = "log2 ratio of SSE, method 1 versus method 2", y = "count")
  savefig(p, "fig-13-7", ch = CH, width = 4.8, height = 3.0)
}

# ===================================================================================================
# Section 13.4.4: Darwin-style paired design, simulated as in the book.
paired_sim <- function(n = 15, effect = 0.2, sdpots = 1, sdnoise = 0.25) {
  pots <- rnorm(n, 0, sdpots)
  hybrid <- pots + effect + rnorm(n, 0, sdnoise)
  autoz <- pots + rnorm(n, 0, sdnoise)
  c(unpaired = t.test(hybrid, autoz, paired = FALSE)$p.value,
    paired   = t.test(hybrid, autoz, paired = TRUE)$p.value)
}

fig_13_10 <- function() {
  set.seed(1310)
  pvs <- replicate(1000, paired_sim())
  d <- tibble(test = rep(c("unpaired (Welch)", "paired"), times = ncol(pvs)), p = as.vector(pvs))
  p <- ggplot(d, aes(x = p, fill = test)) +
    geom_histogram(binwidth = 0.01, boundary = 0, alpha = 0.55, position = "identity") +
    scale_fill_manual(values = unname(COL[c("blue", "orange")])) +
    labs(x = "p-value", y = "count", fill = "test")
  savefig(p, "fig-13-10", ch = CH, width = 5.0, height = 2.8)
}

# ===================================================================================================
# Section 13.4.5: the opinion poll. 100 people sorted by opinion along a street; 12 are polled either
# at random or as 4 random triples of next-door neighbours.
poll_once <- function(n = 100, k = 12) {
  opinion <- sort(rnorm(n))
  i1 <- sample.int(n, k)
  centre <- sample(seq(3, n, by = 3), k / 3)
  i2 <- c(centre, centre - 1, centre - 2)
  c(independent = mean(opinion[i1]), correlated = mean(opinion[i2]))
}

fig_13_12 <- function() {
  set.seed(1312)
  r <- replicate(5000, poll_once())
  d <- tibble(design = rep(rownames(r), times = ncol(r)), value = as.vector(r))
  p <- ggplot(d, aes(x = value, colour = design)) +
    geom_density(linewidth = 0.7) +
    geom_vline(xintercept = 0) +
    scale_colour_manual(values = unname(COL[c("red", "teal")])) +
    labs(x = "opinion poll result", y = "density")
  savefig(p, "fig-13-12", ch = CH, width = 5.0, height = 3.0)
}

# ===================================================================================================
# Data for the demonstrations.
export_data <- function() {
  z <- HistData::ZeaMays
  export_json(list(source = "Darwin (1876), as tabulated by Fisher (1935); HistData::ZeaMays",
                   units = "height in inches", pair = z$pair, pot = z$pot, cross = z$cross, self = z$self),
              "ch13-darwin.json")
}

run_figures(list(fig_13_2 = fig_13_2, fig_13_4 = fig_13_4, fig_13_5 = fig_13_5, fig_13_7 = fig_13_7,
                 fig_13_8 = fig_13_8, fig_13_10 = fig_13_10, fig_13_12 = fig_13_12, export_data = export_data))
