# Chapter 5. Clustering: every figure of Holmes and Huber (2019), Modern Statistics for Modern Biology,
# Chapter 5, that is computed from data or by simulation.
#
#   Rscript ch05_clustering.R              all figures
#   Rscript ch05_clustering.R fig_5_12     one figure
#
# Figures that are NOT computed here (photographs, maps, schematics, cartoons):
#   5.1  John Snow's cholera map (historical map)
#   5.2  bomb impacts in London (map from a website)
#   5.3  breast cancer clustering of Aure et al. (2017) (published figure)
#   5.4  the steps of a clustering analysis (schematic)
#   5.10 the lower triangle of a distance matrix (schematic)
#   5.11 cophenetic distance (cartoon)
#   5.19 Linnaeus' taxonomy (scanned page)
#   5.20 three drawings of the same tree (schematic)
#   5.21, 5.22, 5.23 single linkage, complete linkage, Ward (schematics)
#   5.24 a Calder mobile (photograph)
#   5.26 a tree with 11 leaves (schematic; Question 5.11 is solved in the notes)
#   5.32 bootstrap sampling of clusterings (schematic)
# Figures 5.5-5.8 are equal-distance contour drawings in the book; they are computed here.
# Figure 5.25 was made by the book from a distance matrix (d14) that is not distributed; it is
# re-created on the Morder T-cell data of Section 5.6.
# Figure 5.13 uses the 65 high-coverage cells of the fluidigm data as shipped with the package
# clusterExperiment (the book downloads the same cells through scRNAseq, which is not installed).
# Extra figure: fig-5-ex1, silhouette plot for Exercise 5.1.

source(file.path(local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
}), "common.R"))

suppressPackageStartupMessages({
  library(dplyr)
  library(tibble)
  library(cluster)
})

CH <- 5
CACHE <- file.path(CODE_DIR, "cache")
dir.create(CACHE, showWarnings = FALSE)
cached <- function(name, expr) {
  f <- file.path(CACHE, sprintf("ch05-%s.rds", name))
  if (file.exists(f)) return(readRDS(f))
  val <- force(expr)
  saveRDS(val, f)
  val
}
PAL8 <- unname(c(COL, "#8c564b"))

# =============================================================================================
# Shared data
# =============================================================================================

# Four Gaussian groups of 100 points, centres (0,0), (0,8), (8,0), (8,8), sd 2 (Section 5.7)
make_simdat <- function() {
  set.seed(512)
  g <- expand.grid(my = c(0, 8), mx = c(0, 8))
  bind_rows(lapply(seq_len(nrow(g)), function(i)
    tibble(x = rnorm(100, g$mx[i], 2), y = rnorm(100, g$my[i], 2),
           class = paste(g$mx[i], g$my[i], sep = ":"))))
}

# Two blobs joined by a thin bridge, plus a third blob: shows the chaining of single linkage (demo data)
make_bridge <- function() {
  set.seed(5600)
  a <- cbind(rnorm(20, 0, 0.5), rnorm(20, 0, 0.5))
  b <- cbind(rnorm(20, 5, 0.5), rnorm(20, 0, 0.5))
  br <- cbind(seq(1.1, 3.9, length.out = 8), rnorm(8, 0, 0.12))
  c3 <- cbind(rnorm(12, 2.5, 0.4), rnorm(12, 3.5, 0.4))
  X <- round(rbind(a, b, br, c3), 3)
  data.frame(x = X[, 1], y = X[, 2], group = rep(c("A", "B", "bridge", "C"), c(20, 20, 8, 12)))
}

# Mass cytometry data of Bendall et al. (2011), with marker names (Section 5.5)
load_fcs <- function() {
  suppressPackageStartupMessages(library(flowCore))
  f <- read.FCS(msmb_data("Bendall_2011.fcs"), truncate_max_range = FALSE)
  mk <- read.csv(msmb_data("Bendall_2011_markers.csv"))
  mt <- match(mk$isotope, colnames(f))
  stopifnot(!anyNA(mt))
  colnames(f)[mt] <- mk$marker
  f
}
# asinh(a + x) on every channel except Time, Cell_length and absoluteEventNumber
fcs_asinh <- function(f, a) {
  suppressPackageStartupMessages(library(flowCore))
  transform(f, transformList(colnames(f)[-c(1, 2, 41)], arcsinhTransform(a = a, b = 1)))
}

# Hiiragi2013 embryo data: 50 most variable probes
load_hiiragi <- function() {
  suppressPackageStartupMessages(library(Hiiragi2013))
  e <- new.env(); data("x", package = "Hiiragi2013", envir = e)
  e$x
}
pamfun <- function(x, k) list(cluster = pam(x, k, cluster.only = TRUE))

# =============================================================================================
# 5.3 Distances
# =============================================================================================

contour_fig <- function(dfun, title) {
  g <- expand.grid(x = seq(-2, 2, length.out = 241), y = seq(-2, 2, length.out = 241))
  g$d <- dfun(g$x, g$y)
  ggplot(g, aes(x, y, z = d)) +
    geom_contour(aes(colour = after_stat(level)), breaks = seq(0.25, 2, by = 0.25), linewidth = 0.7) +
    annotate("point", x = 0, y = 0, size = 2) +
    scale_colour_gradient(low = COL[["blue"]], high = COL[["orange"]], guide = "none") +
    coord_fixed(xlim = c(-2, 2), ylim = c(-2, 2), expand = FALSE) +
    labs(x = expression(x[1]), y = expression(x[2]), title = title) +
    theme(plot.title = element_text(size = 11))
}
fig_5_5 <- function() savefig(contour_fig(function(x, y) sqrt(x^2 + y^2), "Euclidean"),
                              "fig-5-5", CH, 3.2, 3.3)
fig_5_6 <- function() {
  S <- matrix(c(1, 0.6, 0.6, 0.5), 2); Si <- solve(S)
  savefig(contour_fig(function(x, y) sqrt(Si[1, 1] * x^2 + 2 * Si[1, 2] * x * y + Si[2, 2] * y^2),
                      "Mahalanobis"), "fig-5-6", CH, 3.2, 3.3)
}
fig_5_7 <- function() savefig(contour_fig(function(x, y) abs(x) + abs(y), "Manhattan (L1)"),
                              "fig-5-7", CH, 3.2, 3.3)
fig_5_8 <- function() savefig(contour_fig(function(x, y) pmax(abs(x), abs(y)), "Maximum (L-infinity)"),
                              "fig-5-8", CH, 3.2, 3.3)

# Ellipse of constant Mahalanobis distance r around mu
mahal_ellipse <- function(mu, S, r, n = 200) {
  t <- seq(0, 2 * pi, length.out = n)
  L <- t(chol(S))
  pts <- t(mu + r * L %*% rbind(cos(t), sin(t)))
  data.frame(x = pts[, 1], y = pts[, 2])
}
fig_5_9 <- function() {
  set.seed(59)
  n <- 0xd000
  S1 <- matrix(c(1, -0.6, -0.6, 1), 2); S2 <- matrix(c(4.5, 2.7, 2.7, 4.5), 2)
  mu1 <- c(0.5, 2.5); mu2 <- c(5, 8); pt <- c(3, 4)
  X <- rbind(MASS::mvrnorm(n, mu1, S1), MASS::mvrnorm(n, mu2, S2))
  pts <- data.frame(x = X[, 1], y = X[, 2])
  ell <- bind_rows(lapply(1:5, function(r) bind_rows(
    cbind(mahal_ellipse(mu1, S1, r), grp = paste0("a", r), r = r),
    cbind(mahal_ellipse(mu2, S2, r), grp = paste0("b", r), r = r))))
  seg <- data.frame(x = pt[1], y = pt[2], xend = c(mu1[1], mu2[1]), yend = c(mu1[2], mu2[2]))
  p <- ggplot(pts, aes(x, y)) +
    geom_bin2d(bins = 150, show.legend = FALSE) +
    scale_fill_gradient(low = "grey85", high = "grey10", trans = "log10") +
    geom_path(data = ell, aes(group = grp, colour = r), linewidth = 0.8) +
    scale_colour_gradient(low = COL[["orange"]], high = COL[["blue"]], name = "Mahalanobis\ndistance") +
    geom_segment(data = seg, aes(xend = xend, yend = yend), colour = COL[["red"]], linewidth = 1) +
    annotate("point", x = pt[1], y = pt[2], colour = COL[["red"]], size = 3) +
    annotate("text", x = mu1[1], y = mu1[2], label = "x", size = 6, colour = "white", fontface = "bold") +
    annotate("text", x = mu2[1], y = mu2[2], label = "+", size = 8, colour = "white", fontface = "bold") +
    coord_fixed(xlim = c(-3, 12), ylim = c(-1, 15)) + labs(x = NULL, y = NULL)
  savefig(p, "fig-5-9", CH, 5, 5, dense = TRUE)
}

# =============================================================================================
# 5.4 k-means and PAM
# =============================================================================================

# Lloyd's algorithm from given centres; returns the assignments and centres after each update
lloyd_path <- function(X, cents, iters) {
  out <- list()
  assign <- function(C) apply(X, 1, function(x) which.min(colSums((t(C) - x)^2)))
  cl <- assign(cents)
  out[[1]] <- list(cl = cl, C = cents)
  for (it in seq_len(iters)) {
    cents <- t(sapply(seq_len(nrow(cents)), function(k) colMeans(X[cl == k, , drop = FALSE])))
    cl <- assign(cents)
    out[[it + 1]] <- list(cl = cl, C = cents)
  }
  out
}
fig_5_12 <- function() {
  set.seed(248811)
  X <- matrix(runif(100), ncol = 2)
  cents <- X[sample(nrow(X), 3), ]
  path <- lloyd_path(X, cents, 3)
  lab <- c("(a) initial centres", "(b) after one update", "(c) after three updates")
  use <- c(1, 2, 4)
  df <- bind_rows(lapply(1:3, function(i) data.frame(x = X[, 1], y = X[, 2],
    cluster = factor(path[[use[i]]]$cl), panel = lab[i])))
  cd <- bind_rows(lapply(1:3, function(i) data.frame(x = path[[use[i]]]$C[, 1], y = path[[use[i]]]$C[, 2],
    panel = lab[i])))
  p <- ggplot(df, aes(x, y)) +
    geom_point(aes(colour = cluster, shape = cluster), size = 2.6) +
    geom_point(data = cd, shape = 1, size = 6, stroke = 1.2) +
    facet_wrap(~panel) + coord_fixed() +
    scale_colour_manual(values = unname(COL[c("blue", "orange", "green")]), guide = "none") +
    scale_shape_manual(values = c(16, 17, 15), guide = "none") + labs(x = NULL, y = NULL)
  savefig(p, "fig-5-12", CH, 8, 3.2)
}

fig_5_13 <- function() {
  ce <- cached("clustermany", {
    suppressPackageStartupMessages(library(clusterExperiment))
    e <- new.env()
    data("fluidigmData", package = "clusterExperiment", envir = e)
    data("fluidigmColData", package = "clusterExperiment", envir = e)
    keep <- e$fluidigmColData$Coverage_Type == "High"
    cnt <- e$fluidigmData$tophat_counts[, keep]
    se <- SummarizedExperiment::SummarizedExperiment(
      assays = list(normalized_counts = as.matrix(limma::normalizeQuantiles(cnt))),
      colData = e$fluidigmColData[keep, ])
    set.seed(513)
    ce <- clusterMany(se, clusterFunction = "pam", ks = c(5, 7, 9), run = TRUE, isCount = TRUE,
                      reduceMethod = "var", nFilterDims = c(60, 100, 150), verbose = FALSE)
    clusterLabels(ce) <- sub("FilterDims", "", clusterLabels(ce))
    ce
  })
  suppressPackageStartupMessages(library(clusterExperiment))
  savefig(function() {
    op <- par(mar = c(1.1, 9, 2.1, 1.1)); on.exit(par(op))
    plotClusters(ce, whichClusters = "workflow", axisLine = -1)
  }, "fig-5-13", CH, 6.4, 4)
}

# =============================================================================================
# 5.5 Flow and mass cytometry
# =============================================================================================

dens_scatter <- function(df, xv, yv, xlab = xv, ylab = yv, bins = 120) {
  ggplot(df, aes(.data[[xv]], .data[[yv]])) +
    geom_bin2d(bins = bins) +
    scale_fill_gradientn(colours = c("#f7fbff", "#6baed6", "#08306b", "#f16913", "#7f2704"),
                         trans = "log10", name = "cells") +
    labs(x = xlab, y = ylab)
}
fig_5_14 <- function() {
  e <- as.data.frame(flowCore::exprs(load_fcs()))
  e <- e[e$DNA191 > 0, ]
  p <- dens_scatter(e, "Cell_length", "DNA191") + scale_y_log10() +
    labs(y = "DNA191 (log scale)")
  savefig(p, "fig-5-14", CH, 5, 4, dense = TRUE)
}
fig_5_15 <- function() {
  f <- load_fcs()
  raw <- flowCore::exprs(f)[, "CD3all"]
  tr <- flowCore::exprs(fcs_asinh(f, 0.1))[, "CD3all"]
  p1 <- ggplot(data.frame(v = raw), aes(v)) + geom_density(fill = COL[["blue"]], alpha = 0.3) +
    labs(x = "CD3all (raw intensity)", y = "density", title = "(a) untransformed")
  p2 <- ggplot(data.frame(v = tr), aes(v)) + geom_density(fill = COL[["orange"]], alpha = 0.3) +
    labs(x = "asinh(0.1 + CD3all)", y = "density", title = "(b) after asinh")
  savefig(gridExtra::arrangeGrob(p1, p2, nrow = 1), "fig-5-15", CH, 8, 3.2)
}
fig_5_16 <- function() {
  suppressPackageStartupMessages(library(flowPeaks))
  e <- flowCore::exprs(fcs_asinh(load_fcs(), 0.1))[, c("CD3all", "CD56")]
  fp <- cached("flowpeaks", { set.seed(516); invisible(capture.output(r <- flowPeaks(e))); r$peaks.cluster })
  df <- data.frame(e, cluster = factor(fp))
  p <- ggplot(df, aes(CD3all, CD56, colour = cluster)) + geom_point(size = 0.15, alpha = 0.35) +
    scale_colour_manual(values = PAL8) +
    guides(colour = guide_legend(override.aes = list(size = 3, alpha = 1))) +
    labs(x = "asinh(0.1 + CD3all)", y = "asinh(0.1 + CD56)")
  savefig(p, "fig-5-16", CH, 5, 4, dense = TRUE)
}
fig_5_17 <- function() {
  e <- as.data.frame(flowCore::exprs(fcs_asinh(load_fcs(), 0.1))[, c("CD3all", "CD56")])
  p <- dens_scatter(e, "CD3all", "CD56", "asinh(0.1 + CD3all)", "asinh(0.1 + CD56)") +
    geom_density_2d(colour = "black", linewidth = 0.3, bins = 12)
  savefig(p, "fig-5-17", CH, 5, 4.2, dense = TRUE)
}
dbscan_mc5 <- function() cached("dbscan-mc5", {
  e <- flowCore::exprs(fcs_asinh(load_fcs(), 0))
  mc5 <- e[, c("CD4", "CD8", "CD20", "CD3all", "CD56")]
  res <- dbscan::dbscan(mc5, eps = 0.65, minPts = 30)
  data.frame(mc5, cluster = factor(res$cluster))
})
fig_5_18 <- function() {
  df <- dbscan_mc5()
  df <- df[df$cluster != "0", ]   # contours of the clusters only (noise points have no contour)
  df$cluster <- droplevels(df$cluster)
  big <- names(which(table(df$cluster) >= 60))
  df <- df[df$cluster %in% big, ]
  mk <- function(xv, yv, ttl) ggplot(df, aes(.data[[xv]], .data[[yv]], colour = cluster)) +
    geom_density_2d(bins = 6, linewidth = 0.4) + scale_colour_manual(values = PAL8) +
    labs(x = paste0("asinh(", xv, ")"), y = paste0("asinh(", yv, ")"), title = ttl)
  g <- gridExtra::arrangeGrob(mk("CD4", "CD8", "(a)"), mk("CD3all", "CD20", "(b)"), nrow = 1)
  savefig(g, "fig-5-18", CH, 9, 3.8)
}

# =============================================================================================
# 5.6 Hierarchical clustering
# =============================================================================================

load_morder <- function() { e <- new.env(); load(msmb_data("Morder.RData"), envir = e); e$Morder }
fig_5_25 <- function() {
  M <- load_morder()
  d <- dist(M)
  cols <- c(EFF = COL[["red"]], MEM = COL[["blue"]], NAI = COL[["green"]])
  savefig(function() {
    op <- par(mfrow = c(1, 3), mar = c(5.2, 4.2, 2.5, 0.5), mgp = c(2.4, 0.7, 0)); on.exit(par(op))
    for (m in c("single", "average", "complete")) {
      h <- hclust(d, method = m)
      dd <- as.dendrogram(h)
      dd <- dendrapply(dd, function(n) {
        if (is.leaf(n)) {
          ct <- substr(attr(n, "label"), 7, 9)
          attr(n, "nodePar") <- list(pch = NA, lab.col = cols[[ct]], lab.cex = 0.75)
        }
        n
      })
      plot(dd, main = sprintf("(%s) %s linkage", letters[match(m, c("single", "average", "complete"))], m),
           ylab = "merge height (Euclidean distance)")
      if (m == "single") legend("topright", legend = c("naive", "effector", "memory"), text.col = cols[c("NAI", "EFF", "MEM")],
                                bty = "n", cex = 0.9)
    }
  }, "fig-5-25", CH, 10, 4.6)
}

# =============================================================================================
# 5.7 Number of clusters
# =============================================================================================

fig_5_27 <- function() {
  sd <- make_simdat()
  p <- ggplot(sd, aes(x, y, colour = class)) + geom_point(size = 1.2) + coord_fixed() +
    scale_colour_manual(values = unname(COL[c("blue", "orange", "green", "purple")]))
  savefig(p, "fig-5-27", CH, 4.6, 3.6)
}
wss_curve <- function(X, kmax = 8, nstart = 25) {
  sapply(1:kmax, function(k) if (k == 1) sum(scale(X, scale = FALSE)^2) else
    kmeans(X, k, nstart = nstart, iter.max = 50)$tot.withinss)
}
fig_5_28 <- function() {
  set.seed(528)
  X <- as.matrix(make_simdat()[, c("x", "y")])
  w <- tibble(k = 1:8, value = wss_curve(X))
  p <- ggplot(w, aes(factor(k), value)) + geom_col(fill = COL[["blue"]]) +
    labs(x = "k", y = expression(WSS[k]))
  savefig(p, "fig-5-28", CH, 3.6, 3.4)
}
fig_5_29 <- function() {
  X <- as.matrix(make_simdat()[, c("x", "y")])
  chv <- tibble(k = 2:8, value = sapply(k, function(i) fpc::calinhara(X, pam(X, i)$clustering)))
  p <- ggplot(chv, aes(k, value)) + geom_line() + geom_point(size = 2, colour = COL[["red"]]) +
    labs(y = "Calinski-Harabasz index")
  savefig(p, "fig-5-29", CH, 3.6, 3.4)
}
plot_gap <- function(gs, marks = NULL) {
  tab <- data.frame(gs$Tab, k = seq_len(nrow(gs$Tab)))
  p <- ggplot(tab, aes(k, gap)) + geom_line() +
    geom_errorbar(aes(ymin = gap - SE.sim, ymax = gap + SE.sim), width = 0.15) +
    geom_point(size = 2.5, colour = COL[["red"]]) + labs(y = "gap(k)")
  if (!is.null(marks)) p <- p + geom_vline(data = marks, aes(xintercept = k, linetype = rule), colour = COL[["blue"]]) +
    scale_linetype_manual(values = c(2, 3), name = NULL) + theme(legend.position = "bottom")
  p
}
fig_5_30 <- function() {
  X <- as.matrix(make_simdat()[, c("x", "y")])
  gs <- cached("gap-simdat", { set.seed(530); clusGap(X, FUNcluster = pamfun, K.max = 8, B = 50, verbose = FALSE) })
  savefig(plot_gap(gs), "fig-5-30", CH, 3.6, 3.8)
}
hiiragi_gap <- function() cached("gap-hiiragi", {
  x <- load_hiiragi()
  ex <- Biobase::exprs(x)
  sel <- order(MatrixGenerics::rowVars(ex), decreasing = TRUE)[1:50]
  embmat <- t(ex[sel, ])
  set.seed(531)
  gs <- clusGap(embmat, FUNcluster = pamfun, K.max = 24, verbose = FALSE)
  k1 <- maxSE(gs$Tab[, "gap"], gs$Tab[, "SE.sim"])
  k2 <- maxSE(gs$Tab[, "gap"], gs$Tab[, "SE.sim"], method = "Tibs2001SEmax")
  cl <- pamfun(embmat, k1)$cluster
  list(gs = gs, k1 = k1, k2 = k2, tab = table(Biobase::pData(x)[names(cl), "sampleGroup"], cl))
})
fig_5_31 <- function() {
  h <- hiiragi_gap()
  marks <- data.frame(k = c(h$k1, h$k2), rule = c(sprintf("firstSEmax: k = %d", h$k1),
                                                  sprintf("Tibs2001SEmax: k = %d", h$k2)))
  savefig(plot_gap(h$gs, marks), "fig-5-31", CH, 5, 4.6)
}
cluster_resampling <- function(x, ngenes = 50, k = 2, B = 250, prob = 0.67) {
  mat <- Biobase::exprs(x)
  ens <- clue::cl_ensemble(list = lapply(seq_len(B), function(b) {
    s <- sample(ncol(mat), size = round(prob * ncol(mat)))
    sub <- mat[, s, drop = FALSE]
    sel <- order(MatrixGenerics::rowVars(sub), decreasing = TRUE)[seq_len(ngenes)]
    fit <- pam(t(sub[sel, , drop = FALSE]), k = k)
    clue::as.cl_partition(clue::cl_predict(fit, t(mat[sel, ]), "memberships"))
  }))
  cons <- clue::cl_consensus(ens)
  list(agreements = sapply(ens, clue::cl_agreement, y = cons), consensus = cons)
}
clue_runs <- function() cached("clue", {
  suppressPackageStartupMessages(library(clue))
  x <- load_hiiragi()
  wt <- x$genotype == "WT"
  set.seed(533)
  r1 <- cluster_resampling(x[, x$Embryonic.day == "E3.25" & wt])
  r2 <- cluster_resampling(x[, x$Embryonic.day == "E3.5" & wt])
  list(ag1 = r1$agreements, ag2 = r2$agreements,
       m1 = clue::cl_membership(r1$consensus)[, 1], m2 = clue::cl_membership(r2$consensus)[, 1])
})
fig_5_33 <- function() {
  r <- clue_runs()
  ag <- bind_rows(tibble(agreements = r$ag1, day = "E3.25"), tibble(agreements = r$ag2, day = "E3.5"))
  p1 <- ggplot(ag, aes(day, agreements)) + geom_boxplot(outlier.shape = NA) +
    ggbeeswarm::geom_quasirandom(width = 0.3, size = 0.8, colour = "#1f5fbf66")
  mem <- bind_rows(tibble(y = sort(r$m1), x = seq_along(y), day = "E3.25"),
                   tibble(y = sort(r$m2), x = seq_along(y), day = "E3.5"))
  p2 <- ggplot(mem, aes(x, y, colour = day)) + geom_point() + facet_grid(~day, scales = "free_x") +
    scale_colour_manual(values = unname(COL[c("orange", "blue")]), guide = "none") +
    labs(x = "sample (sorted)", y = "consensus membership in cluster 1")
  savefig(gridExtra::arrangeGrob(p1, p2, widths = c(2.4, 4)), "fig-5-33", CH, 8, 4)
}

# Extra: silhouette plot for Exercise 5.1
fig_5_ex1 <- function() {
  X <- as.matrix(make_simdat()[, c("x", "y")])
  sil <- silhouette(pam(X, 4))
  df <- data.frame(cluster = factor(sil[, "cluster"]), width = sil[, "sil_width"]) |>
    arrange(cluster, desc(width)) |> mutate(id = row_number())
  p <- ggplot(df, aes(id, width, fill = cluster)) + geom_col(width = 1) + coord_flip() +
    scale_fill_manual(values = unname(COL[c("red", "green", "blue", "purple")])) +
    geom_hline(yintercept = mean(df$width), linetype = 2) +
    labs(x = "observation (sorted within cluster)", y = "silhouette width s(i)",
         title = sprintf("PAM, k = 4: average silhouette width %.3f", mean(df$width))) +
    theme(axis.text.y = element_blank(), axis.ticks.y = element_blank(), plot.title = element_text(size = 10))
  savefig(p, "fig-5-ex1", CH, 4.6, 4.2)
}

# =============================================================================================
# 5.8 Clustering as denoising
# =============================================================================================

fig_5_34 <- function() {
  set.seed(534)
  s1 <- MASS::mvrnorm(1e3, -c(1, 1), 0.5 * diag(2))
  s2 <- MASS::mvrnorm(1e5, c(1, 1), 0.5 * diag(2))
  df <- data.frame(x = c(s1[, 1], s2[, 1]), y = c(s1[, 2], s2[, 2]),
                   seq = factor(rep(c("seq1 (1e3)", "seq2 (1e5)"), c(1e3, 1e5))))
  p <- ggplot(df, aes(x, y, fill = seq)) + geom_hex(bins = 50, alpha = 0.6) + coord_fixed() +
    scale_fill_manual(values = unname(COL[c("orange", "blue")]), name = NULL)
  savefig(p, "fig-5-34", CH, 4.8, 3.8, dense = TRUE)
}
fig_5_35 <- function() {
  set.seed(535)
  n <- 2000; len <- 200; perr <- 0.001
  seqs <- matrix(runif(n * len) >= perr, nrow = n, ncol = len)
  dists <- as.matrix(dist(seqs, method = "manhattan"))
  df <- tibble(k = 10^seq(log10(2), log10(n), length.out = 20),
               diameter = vapply(k, function(i) { s <- sample(n, i); max(dists[s, s]) }, numeric(1)))
  p <- ggplot(df, aes(k, diameter)) + geom_point() + geom_smooth(method = "loess", formula = y ~ x, se = TRUE) +
    labs(x = "number of reads k", y = "diameter (max pairwise mismatches)")
  savefig(p, "fig-5-35", CH, 4.4, 3.3)
}
fig_5_36 <- function() {
  set.seed(536)
  sims <- replicate(1e5, sum(rpois(200, 0.0005)))
  savefig(function() vcd::distplot(sims, type = "poisson", main = ""), "fig-5-36", CH, 4.4, 4)
}
dada_runs <- function() cached("dada", {
  suppressPackageStartupMessages(library(dada2))
  dF <- readRDS(msmb_data("derepFs.rds")); dR <- readRDS(msmb_data("derepRs.rds"))
  set.seed(537)
  ddF <- dada(dF, err = NULL, selfConsist = TRUE, multithread = FALSE, verbose = FALSE)
  ddR <- dada(dR, err = NULL, selfConsist = TRUE, multithread = FALSE, verbose = FALSE)
  dadaFs <- dada(dF, err = ddF[[1]]$err_out, pool = TRUE, multithread = FALSE, verbose = FALSE)
  dadaRs <- dada(dR, err = ddR[[1]]$err_out, pool = TRUE, multithread = FALSE, verbose = FALSE)
  mergers <- mergePairs(dadaFs, dF, dadaRs, dR)
  st.all <- makeSequenceTable(mergers[!grepl("Mock", names(mergers))])
  st <- removeBimeraDenovo(st.all, multithread = FALSE)
  list(ddF1 = ddF[[1]], err_out = ddF[[1]]$err_out, seqtab.all = st.all, seqtab = st,
       nF = length(dadaFs), nR = length(dadaRs))
})
fig_5_37 <- function() {
  suppressPackageStartupMessages(library(dada2))
  r <- dada_runs()
  p <- suppressWarnings(plotErrors(r$ddF1, nominalQ = TRUE))
  savefig(function() suppressWarnings(print(p)), "fig-5-37", CH, 8, 6)
}
fig_5_38 <- function() {
  suppressPackageStartupMessages(library(kernlab))
  e <- new.env(); data("spirals", package = "kernlab", envir = e); sp <- e$spirals
  set.seed(538)
  km <- kmeans(sp, 2, nstart = 10)$cluster
  db <- dbscan::dbscan(sp, eps = 0.16, minPts = 3)$cluster
  df <- rbind(data.frame(x = sp[, 1], y = sp[, 2], cl = factor(km), panel = "(a) k-means, k = 2"),
              data.frame(x = sp[, 1], y = sp[, 2], cl = factor(db), panel = "(b) dbscan, eps = 0.16, minPts = 3"))
  p <- ggplot(df, aes(x, y, colour = cl)) + geom_point(size = 1.2) + facet_wrap(~panel) + coord_fixed() +
    scale_colour_manual(values = c(`0` = "grey60", `1` = COL[["blue"]], `2` = COL[["red"]], `3` = COL[["green"]]),
                        name = "cluster") + labs(x = NULL, y = NULL)
  savefig(p, "fig-5-38", CH, 8, 3.8)
}
fig_5_39 <- function() {
  suppressPackageStartupMessages(library(dada2))
  mp <- msmb_data("MiSeq_SOP")
  fnF <- sort(list.files(mp, pattern = "_R1_001.fastq$", full.names = TRUE))
  fnR <- sort(list.files(mp, pattern = "_R2_001.fastq$", full.names = TRUE))
  p1 <- plotQualityProfile(fnF[1:2]) + ggtitle("(a) forward reads")
  p2 <- plotQualityProfile(fnR[1:2]) + ggtitle("(b) reverse reads")
  savefig(gridExtra::arrangeGrob(p1, p2, nrow = 1), "fig-5-39", CH, 10, 4.4, dense = TRUE)
}

# =============================================================================================
# Data for the browser demonstrations
# =============================================================================================

export_demo_data <- function() {
  sd <- make_simdat()
  set.seed(540)
  un <- data.frame(x = runif(400, min(sd$x), max(sd$x)), y = runif(400, min(sd$y), max(sd$y)))
  e <- new.env(); data("spirals", package = "kernlab", envir = e); sp <- e$spirals
  br <- make_bridge()
  export_json(list(
    simdat = list(x = round(sd$x, 4), y = round(sd$y, 4), class = sd$class),
    uniform = list(x = round(un$x, 4), y = round(un$y, 4)),
    spirals = list(x = round(sp[, 1], 5), y = round(sp[, 2], 5)),
    bridge = list(x = br$x, y = br$y, group = br$group)
  ), "ch05-points.json")

  # Hiiragi2013: 101 embryo cells x 50 most variable probes (rounded to 3 decimals)
  x <- load_hiiragi(); ex <- Biobase::exprs(x)
  sel <- order(MatrixGenerics::rowVars(ex), decreasing = TRUE)[1:50]
  export_json(list(group = as.character(Biobase::pData(x)$sampleGroup),
                   X = unname(round(t(ex[sel, ]), 3))), "ch05-hiiragi.json")

  # Morder (30 T-cell samples x 156 genes), rounded to 3 decimals
  M <- load_morder()
  export_json(list(samples = rownames(M), celltype = substr(rownames(M), 7, 9),
                   status = substr(rownames(M), 1, 3), X = unname(round(M, 3))), "ch05-morder.json")

  # 5000 cells of the Bendall data, raw intensities for seven markers (rounded to 2 decimals)
  f <- flowCore::exprs(load_fcs())
  set.seed(541)
  idx <- sort(sample(nrow(f), 5000))
  mk <- c("CD3all", "CD56", "CD4", "CD8", "CD20", "CD45RA", "CD33")
  export_json(c(list(n = length(idx), markers = mk),
                setNames(lapply(mk, function(m) round(unname(f[idx, m]), 2)), mk)), "ch05-cytof.json")
}

run_figures(list(
  fig_5_5 = fig_5_5, fig_5_6 = fig_5_6, fig_5_7 = fig_5_7, fig_5_8 = fig_5_8, fig_5_9 = fig_5_9,
  fig_5_12 = fig_5_12, fig_5_13 = fig_5_13, fig_5_14 = fig_5_14, fig_5_15 = fig_5_15, fig_5_16 = fig_5_16,
  fig_5_17 = fig_5_17, fig_5_18 = fig_5_18, fig_5_25 = fig_5_25, fig_5_27 = fig_5_27, fig_5_28 = fig_5_28,
  fig_5_29 = fig_5_29, fig_5_30 = fig_5_30, fig_5_31 = fig_5_31, fig_5_33 = fig_5_33, fig_5_ex1 = fig_5_ex1,
  fig_5_34 = fig_5_34, fig_5_35 = fig_5_35, fig_5_36 = fig_5_36, fig_5_37 = fig_5_37, fig_5_38 = fig_5_38,
  fig_5_39 = fig_5_39, export_demo_data = export_demo_data
))
