"""James et al. (2021, 2023), Chapter 13: Multiple Testing.

Reproduces the figures of Chapter 13 of G. James, D. Witten, T. Hastie and R. Tibshirani, An
Introduction to Statistical Learning with Applications in R, 2nd edition (Springer, 2021), whose
text and figures are shared with the Python edition (James, Witten, Hastie, Tibshirani and Taylor,
2023). Written from scratch with numpy, scipy and matplotlib; every method is a small function meant
to be read next to the lecture notes (../ch13-multiple-testing.html):

* Tests: t_one_sample (the one-sample t-test of Table 13.3), t_two_sample (the pooled two-sample
  t-statistic (13.1)-(13.2) = (13.11), for many variables at once), contrast_test (a linear
  combination of group means, with the pooled ANOVA variance, as in (13.8)).
* Section 13.3, the family-wise error rate: fwer_independent ((13.5)), bonferroni, holm (Algorithm
  13.1) with adjusted p-values p_holm, sidak, tukey_alpha and tukey_hsd (Tukey's method via the
  studentized range distribution), scheffe_alpha (Scheffe's threshold), power_vs_fwer (Figure 13.5,
  computed exactly rather than simulated).
* Section 13.4, the false discovery rate: bh (Algorithm 13.2) with q-values p_bh, bh_estimated_fdr
  (the estimate m p_(k) / k behind it).
* Section 13.5, re-sampling: perm_pvalue (Algorithm 13.3 for one variable), khan_permutations (one
  set of B random relabellings applied to all 2,308 genes at once, the efficient form of Algorithm
  13.4 described in the book's footnote), plugin_fdr (Algorithm 13.4) and pooled_pvalues ((13.14)).

Figures reproduced: 13.1 to 13.11 (all of the chapter's figures; Tables 13.1, 13.2 are schematic
and Table 13.3 is printed in the notes from fund_table()). Run `python3 ch13_multiple_testing.py
--list`, `python3 ch13_multiple_testing.py` (all figures, about a minute the first time, when the
10,000 permutations of the Khan data are computed and cached) or `--fig 13.6`. Files go to
../img/ch13/; `--fig data` writes the demonstrations' data to ../data/.

Data: Fund (2,000 fund managers, 50 months of excess returns) is read by common.isl_data(). Khan is
an R data set of the ISLR2 package and has no CSV file on the book's website; khan() exports it once
with Rscript to cache/ch13-Khan.csv (training and test sets stacked, 83 x 2,308, with the class y).

Settings chosen where the book gives none:
* Figure 13.3: m = 10 two-sided z-tests, 8 false nulls with mean 3.5 and 2 true nulls; the three
  panels are numpy default_rng seeds 1, 31 and 300 (chosen to show the three cases the book describes:
  Holm rejects as many as Bonferroni, one more, and four more).
* Figure 13.4: G = 6 groups of n = 10 N(mu_g, 1) observations, mu_1 = ... = mu_5 = 0, mu_6 = 1.5;
  the 15 pairwise t-tests use the pooled ANOVA variance (54 degrees of freedom), as Tukey's method
  does; seeds 128, 3 and 2 (Tukey rejects two more, one more, and as many as Bonferroni).
* Figure 13.5: m two-sided z-tests, 90% true nulls, false nulls with mean 3; each test is run at
  level a, so the FWER is 1 - (1 - a)^m0 exactly and the power is P(|Z + 3| > z_(a/2)); the curves
  trace a from 1e-8 to 0.5. Nothing is simulated.
* Figures 13.7-13.9 and 13.11: one set of B = 10,000 random relabellings of the 54 patients
  (default_rng(2021)) is applied to every gene; Figures 13.7 and 13.8 are the 10,000 permuted
  statistics of genes 11 and 877, Figure 13.11 the plug-in FDR for the 100 genes that the R lab draws
  with set.seed(1); sample(2308, 100).
"""
import json
import os
import subprocess

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import numpy as np
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt

import common
from common import BLUE, GREEN, GREY, ORANGE, PURPLE, RED

CH = 13
HERE = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(HERE, "cache")
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
KHAN_CSV = os.path.join(CACHE_DIR, "ch13-Khan.csv")
PERM_NPZ = os.path.join(CACHE_DIR, "ch13-khan-perm.npz")
B_PERM, PERM_SEED = 10000, 2021

# the 100 genes of the R lab: set.seed(1); sample(2308, 100) in R 4.x (1-based gene numbers)
R_LAB_GENES = np.array([
    1017, 679, 2177, 930, 1533, 471, 270, 1211, 597, 1301, 1974, 330, 1799, 1615, 1749, 37, 1129, 729, 878, 485,
    1826, 975, 554, 1446, 2159, 1948, 1530, 343, 40, 537, 248, 1222, 2087, 1414, 1304, 1696, 526, 1069, 1426, 22,
    1742, 1766, 2241, 1395, 1128, 983, 1791, 1516, 1640, 1639, 843, 465, 996, 1549, 1200, 1134, 84, 1895, 1165, 2283,
    1328, 557, 1685, 287, 1522, 858, 1840, 576, 990, 316, 1075, 733, 1314, 2306, 811, 955, 1167, 1706, 501, 536,
    988, 1067, 1025, 29, 1942, 838, 1820, 1652, 1317, 573, 1966, 369, 86, 1507, 1646, 355, 1843, 1073, 361, 1340])


# ======================================================================== data
def fund():
    """The Fund data: a 50 x 2000 DataFrame of monthly excess returns (columns Manager1, ...)."""
    return common.isl_data("Fund", index_col=0)


def khan():
    """Khan gene expression data (ISLR2): X (83 x 2308, training rows then test rows) and the class y."""
    if not os.path.exists(KHAN_CSV):
        os.makedirs(CACHE_DIR, exist_ok=True)
        code = ('library(ISLR2); x <- rbind(Khan$xtrain, Khan$xtest); y <- c(Khan$ytrain, Khan$ytest); '
                'd <- data.frame(y = y, x); colnames(d) <- c("y", sprintf("G%04d", 1:2308)); '
                'write.csv(format(d, digits = 15), "' + KHAN_CSV + '", row.names = FALSE, quote = FALSE)')
        subprocess.run(["Rscript", "-e", code], check=True)
    d = pd.read_csv(KHAN_CSV)
    return d.drop(columns="y").to_numpy(float), d["y"].to_numpy(int)


def khan_pair():
    """The two classes compared in Section 13.5: class 2 (29 patients) and class 4 (25 patients)."""
    X, y = khan()
    return X[y == 2], X[y == 4]


# ======================================================================== tests
def t_one_sample(X, mu=0.0):
    """One-sample t-statistics sqrt(n) (xbar - mu) / s and two-sided p-values, column by column."""
    X = np.asarray(X, float)
    n = X.shape[0]
    t = np.sqrt(n) * (X.mean(0) - mu) / X.std(0, ddof=1)
    return t, 2 * stats.t.sf(np.abs(t), n - 1)


def t_two_sample(x1, x2):
    """Pooled two-sample t-statistic (13.1)-(13.2), column by column, with its two-sided t p-value."""
    x1, x2 = np.asarray(x1, float), np.asarray(x2, float)
    n1, n2 = len(x1), len(x2)
    s2 = ((n1 - 1) * x1.var(0, ddof=1) + (n2 - 1) * x2.var(0, ddof=1)) / (n1 + n2 - 2)
    t = (x1.mean(0) - x2.mean(0)) / np.sqrt(s2 * (1 / n1 + 1 / n2))
    return t, 2 * stats.t.sf(np.abs(t), n1 + n2 - 2)


def contrast_test(groups, c):
    """t-test of sum_g c_g mu_g = 0 with the pooled within-group variance (df = N - G)."""
    groups = [np.asarray(g, float) for g in groups]
    c = np.asarray(c, float)
    N, G = sum(len(g) for g in groups), len(groups)
    s2 = sum(((g - g.mean()) ** 2).sum() for g in groups) / (N - G)
    est = sum(cg * g.mean() for cg, g in zip(c, groups))
    se = np.sqrt(s2 * sum(cg ** 2 / len(g) for cg, g in zip(c, groups)))
    t = est / se
    return t, 2 * stats.t.sf(abs(t), N - G), N - G


# ======================================================================== FWER
def fwer_independent(alpha, m):
    """(13.5): the FWER of m independent tests of true nulls, each at level alpha."""
    return 1 - (1 - np.asarray(alpha, float)) ** np.asarray(m, float)


def bonferroni(p, alpha):
    p = np.asarray(p, float)
    return p <= alpha / len(p)


def p_bonferroni(p):
    p = np.asarray(p, float)
    return np.minimum(1, len(p) * p)


def holm(p, alpha):
    """Algorithm 13.1: reject H_(1), ..., H_(L-1), L the first j with p_(j) > alpha / (m + 1 - j)."""
    p = np.asarray(p, float)
    m = len(p)
    o = np.argsort(p, kind="stable")
    above = p[o] > alpha / (m + 1 - np.arange(1, m + 1))
    L = int(np.argmax(above)) + 1 if above.any() else m + 1          # 1-based; m + 1 means "reject all"
    reject = np.zeros(m, bool)
    reject[o[:L - 1]] = True
    return reject


def p_holm(p):
    """Holm-adjusted p-values: running maximum of (m + 1 - j) p_(j), capped at 1."""
    p = np.asarray(p, float)
    m = len(p)
    o = np.argsort(p, kind="stable")
    adj = np.minimum(1, np.maximum.accumulate((m - np.arange(m)) * p[o]))
    out = np.empty(m)
    out[o] = adj
    return out


def sidak(alpha, m):
    """The per-test level that gives FWER exactly alpha for m independent tests."""
    return 1 - (1 - alpha) ** (1.0 / m)


def tukey_alpha(G, df, alpha=0.05):
    """Tukey's per-comparison p-value threshold alpha_T for the G(G-1)/2 pairwise t-tests (pooled df)."""
    q = stats.studentized_range.ppf(1 - alpha, G, df)
    return 2 * stats.t.sf(q / np.sqrt(2), df)


def tukey_hsd(groups, alpha=0.05):
    """Tukey's honest significant differences for equal or unequal group sizes (Tukey-Kramer).

    Returns a DataFrame with one row per pair (j, k), j < k: the difference mean_k - mean_j, the
    simultaneous (1 - alpha) interval and the adjusted p-value, as R's TukeyHSD() prints them."""
    groups = [np.asarray(g, float) for g in groups]
    G, N = len(groups), sum(len(g) for g in groups)
    df = N - G
    s2 = sum(((g - g.mean()) ** 2).sum() for g in groups) / df
    q = stats.studentized_range.ppf(1 - alpha, G, df)
    rows = []
    for j in range(G):
        for k in range(j + 1, G):
            d = groups[k].mean() - groups[j].mean()
            se = np.sqrt(s2 / 2 * (1 / len(groups[j]) + 1 / len(groups[k])))
            rows.append({"pair": "%d-%d" % (k + 1, j + 1), "diff": d, "lwr": d - q * se, "upr": d + q * se,
                         "p_adj": stats.studentized_range.sf(abs(d) / se, G, df)})
    return pd.DataFrame(rows)


def scheffe_alpha(G, df, alpha=0.05):
    """Scheffe's p-value threshold: reject a contrast if |t| > sqrt((G - 1) F_(1-alpha; G-1, df))."""
    c = np.sqrt((G - 1) * stats.f.ppf(1 - alpha, G - 1, df))
    return 2 * stats.t.sf(c, df), c


def power_vs_fwer(m, delta=3.0, frac_null=0.9, a=None):
    """Figure 13.5, exactly: m two-sided z-tests at per-test level a, m0 = 0.9 m true nulls, false nulls
    with mean delta. Returns (FWER, power) along the grid of levels a."""
    a = np.logspace(-8, np.log10(0.5), 400) if a is None else np.asarray(a, float)
    m0 = int(round(frac_null * m))
    z = stats.norm.isf(a / 2)
    return 1 - (1 - a) ** m0, stats.norm.sf(z - delta) + stats.norm.cdf(-z - delta)


# ======================================================================== FDR
def bh(p, q):
    """Algorithm 13.2 (Benjamini-Hochberg): reject every p_j <= p_(L), L = max{j : p_(j) <= q j / m}."""
    p = np.asarray(p, float)
    m = len(p)
    ps = np.sort(p)
    ok = np.nonzero(ps <= q * np.arange(1, m + 1) / m)[0]
    if len(ok) == 0:
        return np.zeros(m, bool)
    return p <= ps[ok.max()]


def p_bh(p):
    """BH-adjusted p-values (q-values): q_(j) = min over k >= j of m p_(k) / k, capped at 1."""
    p = np.asarray(p, float)
    m = len(p)
    o = np.argsort(p, kind="stable")
    raw = m * p[o] / np.arange(1, m + 1)
    adj = np.minimum(1, np.minimum.accumulate(raw[::-1])[::-1])
    out = np.empty(m)
    out[o] = adj
    return out


def bh_estimated_fdr(p):
    """m p_(k) / k for k = 1..m: the estimated FDR of rejecting the k smallest p-values."""
    ps = np.sort(np.asarray(p, float))
    return len(ps) * ps / np.arange(1, len(ps) + 1)


# ======================================================================== re-sampling
def perm_pvalue(x1, x2, B, rng):
    """Algorithm 13.3 for one variable: the fraction of B random relabellings with |T*| >= |T|."""
    z = np.concatenate([x1, x2])
    n1 = len(x1)
    T = t_two_sample(x1, x2)[0]
    Ts = np.empty(B)
    for b in range(B):
        zb = rng.permutation(z)
        Ts[b] = t_two_sample(zb[:n1], zb[n1:])[0]
    return np.mean(np.abs(Ts) >= abs(T)), T, Ts


def perm_tstats(Z, n1, groups):
    """t-statistics (13.11) of all columns of Z for each relabelling in `groups` (b x n boolean, True =
    first group): one matrix product gives the group sums of every gene for every relabelling."""
    n, n2 = Z.shape[0], Z.shape[0] - n1
    Zc = Z - Z.mean(0)                                   # centring changes no t-statistic, avoids cancellation
    G = groups.astype(float)
    s1, q1 = G @ Zc, G @ (Zc ** 2)
    S, Q = Zc.sum(0), (Zc ** 2).sum(0)
    m1, m2 = s1 / n1, (S - s1) / n2
    ss = (q1 - n1 * m1 ** 2) + (Q - q1 - n2 * m2 ** 2)
    s2 = ss / (n - 2)
    return (m1 - m2) / np.sqrt(s2 * (1 / n1 + 1 / n2))


def khan_permutations(B=B_PERM, seed=PERM_SEED, batch=500, refresh=False):
    """B random relabellings of the 54 patients, applied to all 2,308 genes (cached).

    Returns a dict with T (observed statistics), p (t_52 p-values), c (the observed |T| sorted
    increasingly), N (N[k] = number of the B * m permuted |T*| that are >= c[k]), N100 (the same for
    the 100 genes of the R lab, against their own sorted |T|), Nj (Nj[j] = number of the B relabellings
    with |T*_j| >= |T_j|, so Nj / B is the permutation p-value (13.12) of gene j) and the permuted
    statistics of genes 11 and 877 (T11, T877)."""
    if os.path.exists(PERM_NPZ) and not refresh:
        z = np.load(PERM_NPZ)
        if int(z["B"]) == B and int(z["seed"]) == seed and "Nj" in z.files:
            return {k: z[k] for k in z.files}
    x1, x2 = khan_pair()
    Z = np.vstack([x1, x2])
    n, n1 = Z.shape[0], len(x1)
    T, p = t_two_sample(x1, x2)
    c = np.sort(np.abs(T))
    sub = R_LAB_GENES - 1
    c100 = np.sort(np.abs(T[sub]))
    cnt, cnt100, Nj = np.zeros(len(c) + 1), np.zeros(len(c100) + 1), np.zeros(len(T))
    T11, T877 = [], []
    rng = np.random.default_rng(seed)
    done = 0
    while done < B:
        b = min(batch, B - done)
        perms = np.argsort(rng.random((b, n)), axis=1)
        groups = np.zeros((b, n), bool)
        np.put_along_axis(groups, perms[:, :n1], True, axis=1)
        Ts = perm_tstats(Z, n1, groups)
        A = np.abs(Ts)
        cnt += np.bincount(np.searchsorted(c, A.ravel(), side="right"), minlength=len(c) + 1)
        cnt100 += np.bincount(np.searchsorted(c100, A[:, sub].ravel(), side="right"), minlength=len(c100) + 1)
        Nj += (A >= np.abs(T)).sum(0)
        T11.append(Ts[:, 10])
        T877.append(Ts[:, 876])
        done += b
    # N[k] = #{|T*| >= c[k]} = number of values whose insertion index is > k
    N = cnt[::-1].cumsum()[::-1][1:]
    N100 = cnt100[::-1].cumsum()[::-1][1:]
    out = {"T": T, "p": p, "c": c, "N": N, "c100": c100, "N100": N100, "Nj": Nj, "T11": np.concatenate(T11),
           "T877": np.concatenate(T877), "B": np.array(B), "seed": np.array(seed)}
    os.makedirs(CACHE_DIR, exist_ok=True)
    np.savez_compressed(PERM_NPZ, **out)
    return out


def plugin_fdr(c, N, B):
    """Algorithm 13.4 at the thresholds c[k] (sorted increasingly, no ties): R = number of |T| >= c[k],
    V-hat = N[k] / B, FDR-hat = V-hat / R. Returned in order of increasing R."""
    m = len(c)
    R = m - np.arange(m)
    V = N / B
    return R[::-1], (V / R)[::-1]


def pooled_pvalues(T, c, N, B):
    """(13.14): p_j = #{permuted |T*| over all genes >= |T_j|} / (B m)."""
    m = len(T)
    k = np.searchsorted(c, np.abs(T), side="left")     # c[k] == |T_j|
    return N[k] / (B * m)


# ======================================================================== helpers
def fund_table(k=5):
    """Table 13.3: mean, standard deviation, t-statistic and p-value of the first k managers."""
    F = fund().iloc[:, :k]
    t, p = t_one_sample(F)
    return pd.DataFrame({"mean": F.mean().to_numpy(), "sd": F.std(ddof=1).to_numpy(), "t": t, "p": p},
                        index=F.columns)


def holm_bonf_sim(seed, m=10, m0=2, mu=3.5):
    """One panel of Figure 13.3: p-values of m two-sided z-tests, the first m - m0 false nulls."""
    rng = np.random.default_rng(seed)
    z = rng.standard_normal(m) + np.r_[np.full(m - m0, mu), np.zeros(m0)]
    return 2 * stats.norm.sf(np.abs(z)), np.r_[np.ones(m - m0, bool), np.zeros(m0, bool)]


def tukey_sim(seed, G=6, n=10, delta=1.5):
    """One panel of Figure 13.4: pairwise pooled t-test p-values for G groups, the last mean shifted."""
    rng = np.random.default_rng(seed)
    groups = [rng.standard_normal(n) + (delta if g == G - 1 else 0.0) for g in range(G)]
    df = G * (n - 1)
    s2 = sum(((g - g.mean()) ** 2).sum() for g in groups) / df
    p, false = [], []
    for j in range(G):
        for k in range(j + 1, G):
            t = (groups[j].mean() - groups[k].mean()) / np.sqrt(s2 * 2 / n)
            p.append(2 * stats.t.sf(abs(t), df))
            false.append(k == G - 1)
    return np.array(p), np.array(false), df


def _logp_axis(ax):
    ax.set_yscale("log")
    ax.set_xlabel("Ordering of p-values")
    ax.set_ylabel("p-value (log scale)")


# ======================================================================== figures
def fig_13_1(out=None):
    """The N(0, 1) density and the two-sided p-value of T = 2.33."""
    fig, ax = plt.subplots(figsize=(5.0, 3.3))
    x = np.linspace(-4.2, 4.2, 600)
    ax.plot(x, stats.norm.pdf(x), color="k", lw=1.6)
    for lo, hi in ((2.33, 4.2), (-4.2, -2.33)):
        xs = np.linspace(lo, hi, 200)
        ax.fill_between(xs, stats.norm.pdf(xs), color=ORANGE, alpha=0.55, lw=0)
    ax.axvline(2.33, color=RED, lw=1.6)
    ax.text(2.45, 0.30, "T = 2.33", color=RED)
    ax.text(3.0, 0.06, "1%", ha="center", fontsize=9)
    ax.text(-3.0, 0.06, "1%", ha="center", fontsize=9)
    ax.set(xlabel="Value of test statistic", ylabel="Probability density", xlim=(-4.2, 4.2), ylim=(0, 0.42))
    common.savefig(fig, CH, "fig13-1", out)


def fig_13_2(out=None):
    """The FWER (13.5) against the number of independent tests, for alpha = 0.05, 0.01, 0.001."""
    m = np.arange(1, 501)
    fig, ax = plt.subplots(figsize=(5.2, 3.6))
    for a, col in ((0.05, ORANGE), (0.01, BLUE), (0.001, PURPLE)):
        ax.plot(m, fwer_independent(a, m), color=col, label=r"$\alpha = %g$" % a)
    ax.axhline(0.05, color="k", ls="--", lw=1)
    ax.set_xscale("log")
    ax.set_xticks([1, 2, 5, 10, 20, 50, 100, 200, 500], ["1", "2", "5", "10", "20", "50", "100", "200", "500"])
    ax.set(xlabel="Number of hypotheses", ylabel="Family-wise error rate", ylim=(0, 1.02))
    ax.legend(loc="upper left", frameon=False)
    common.savefig(fig, CH, "fig13-2", out)


FIG3_SEEDS, FIG4_SEEDS = (1, 31, 300), (128, 3, 2)


def fig_13_3(out=None):
    """Bonferroni and Holm on three simulated sets of m = 10 p-values, 2 of them true nulls."""
    fig, axes = plt.subplots(1, 3, figsize=(9.6, 3.5))
    alpha, m = 0.05, 10
    for ax, seed in zip(axes, FIG3_SEEDS):
        p, false = holm_bonf_sim(seed)
        o = np.argsort(p)
        j = np.arange(1, m + 1)
        ax.scatter(j, p[o], c=[RED if f else "k" for f in false[o]], s=22, zorder=3)
        ax.axhline(alpha / m, color="k", lw=1.2)
        ax.plot(j, alpha / (m + 1 - j), color=BLUE, lw=1.3)
        _logp_axis(ax)
        nb, nh = bonferroni(p, alpha).sum(), holm(p, alpha).sum()
        ax.set_title("Bonferroni rejects %d, Holm %d" % (nb, nh), fontsize=9.5)
        ax.set_xticks([2, 4, 6, 8, 10])
    common.savefig(fig, CH, "fig13-3", out)


def fig_13_4(out=None):
    """Bonferroni, Tukey and no correction for the 15 pairwise comparisons of G = 6 means."""
    fig, axes = plt.subplots(1, 3, figsize=(9.6, 3.5))
    alpha = 0.05
    for ax, seed in zip(axes, FIG4_SEEDS):
        p, false, df = tukey_sim(seed)
        aT = tukey_alpha(6, df, alpha)
        o = np.argsort(p)
        j = np.arange(1, len(p) + 1)
        ax.scatter(j, p[o], c=[RED if f else "k" for f in false[o]], s=20, zorder=3)
        ax.axhline(alpha / 15, color="k", lw=1.2)
        ax.axhline(aT, color=BLUE, lw=1.3)
        ax.axhline(alpha, color=GREEN, lw=1.3)
        _logp_axis(ax)
        ax.set_title("Bonferroni rejects %d, Tukey %d" % ((p <= alpha / 15).sum(), (p <= aT).sum()), fontsize=9.5)
    common.savefig(fig, CH, "fig13-4", out)


def fig_13_5(out=None):
    """Power against FWER for m = 10, 100, 500 (90% true nulls), computed exactly."""
    fig, ax = plt.subplots(figsize=(4.6, 3.9))
    for m, col in ((10, ORANGE), (100, BLUE), (500, PURPLE)):
        fw, pw = power_vs_fwer(m)
        ax.plot(fw, pw, color=col, label="m = %d" % m)
    ax.axvline(0.05, color="k", ls="--", lw=1)
    ax.set(xlabel="Family-wise error rate", ylabel="Power", xlim=(0, 1), ylim=(0, 1))
    ax.legend(loc="lower right", frameon=False)
    common.savefig(fig, CH, "fig13-5", out)


def fig_13_6(out=None):
    """The 2,000 ordered Fund p-values with the Bonferroni and BH thresholds at 0.05, 0.1, 0.3."""
    F = fund()
    p = t_one_sample(F)[1]
    ps = np.sort(p)
    m = len(p)
    j = np.arange(1, m + 1)
    fig, axes = plt.subplots(1, 3, figsize=(9.8, 3.5))
    for ax, a in zip(axes, (0.05, 0.1, 0.3)):
        rej = bh(p, a)
        R = rej.sum()
        ax.scatter(j, ps, s=5, c=["#3a6fd8" if k < R else GREY for k in range(m)], lw=0)
        ax.plot(j, a * j / m, color=ORANGE, lw=1.6)
        ax.axhline(a / m, color=GREEN, lw=1.6)
        ax.set(xscale="log", yscale="log", xlabel="Index", ylabel="p-value", ylim=(5e-6, 1.5))
        ax.set_title(r"$\alpha = q = %g$: BH rejects %d, Bonferroni %d" % (a, R, bonferroni(p, a).sum()), fontsize=9)
    common.savefig(fig, CH, "fig13-6", out)


def _null_hist(Ts, T, gene, out, name):
    fig, ax = plt.subplots(figsize=(5.2, 3.6))
    ax.hist(Ts, bins=np.linspace(-4.2, 4.2, 85), density=True, color="#f2c14e", edgecolor="white", lw=0.3)
    x = np.linspace(-4.2, 4.2, 600)
    ax.plot(x, stats.t.pdf(x, 52), color=RED, lw=2)
    ax.axvline(T, color=BLUE, lw=2)
    ax.text(T + 0.15, 0.42, "T = %.4f" % T, color=BLUE)
    ax.set(xlim=(-4.2, 4.2), ylim=(0, 0.48), xlabel="Null distribution of test statistic for gene %d" % gene,
           ylabel="Density")
    common.savefig(fig, CH, name, out)


def fig_13_7(out=None):
    """Permutation null distribution of gene 11 against the t_52 density."""
    P = khan_permutations()
    _null_hist(P["T11"], P["T"][10], 11, out, "fig13-7")


def fig_13_8(out=None):
    """Permutation null distribution of gene 877 against the t_52 density."""
    P = khan_permutations()
    _null_hist(P["T877"], P["T"][876], 877, out, "fig13-8")


def fig_13_9(out=None):
    """Estimated FDR against the number of rejections: BH (theoretical p-values) and Algorithm 13.4."""
    P = khan_permutations()
    R, fdr = plugin_fdr(P["c"], P["N"], int(P["B"]))
    fig, ax = plt.subplots(figsize=(5.0, 3.6))
    ax.plot(R, np.minimum(fdr, 1), color=BLUE, lw=2.2, label="re-sampling (Algorithm 13.4)")
    ax.plot(np.arange(1, len(P["p"]) + 1), np.minimum(bh_estimated_fdr(P["p"]), 1), color=ORANGE, lw=1.8, ls="--",
            label=r"Benjamini-Hochberg, $m\,p_{(k)}/k$")
    ax.set(xlabel="Number of rejections", ylabel="False discovery rate", ylim=(0, 1.02))
    ax.legend(loc="upper left", frameon=False, fontsize=8.5)
    common.savefig(fig, CH, "fig13-9", out)


def fig_13_10(out=None):
    """Tukey's simultaneous 95% intervals for the ten pairwise differences of the first five managers."""
    F = fund().iloc[:, :5]
    tab = tukey_hsd([F.iloc[:, g] for g in range(5)])
    order = ["2-1", "3-1", "4-1", "5-1", "3-2", "4-2", "5-2", "4-3", "5-3", "5-4"]
    tab = tab.set_index("pair").loc[order]
    fig, ax = plt.subplots(figsize=(5.0, 4.0))
    y = np.arange(len(tab))[::-1]
    ax.hlines(y, tab["lwr"], tab["upr"], color="k", lw=1.4)
    ax.plot(tab["diff"], y, "o", color=BLUE, ms=5)
    ax.axvline(0, color=GREY, ls="--", lw=1)
    ax.set_yticks(y, tab.index)
    ax.set(xlabel="Differences in mean returns", title="95% family-wise confidence level", xlim=(-7.5, 7.5))
    common.savefig(fig, CH, "fig13-10", out)


def fig_13_11(out=None):
    """Plug-in FDR (Algorithm 13.4) for the 100 genes of the R lab."""
    P = khan_permutations()
    R, fdr = plugin_fdr(P["c100"], P["N100"], int(P["B"]))
    fig, ax = plt.subplots(figsize=(5.0, 3.6))
    ax.plot(R, fdr, color=BLUE, lw=2.4)
    ax.set(xlabel="Number of rejections", ylabel="False discovery rate", ylim=(0, 1.02))
    common.savefig(fig, CH, "fig13-11", out)


# ======================================================================== demonstrations' data
def export_demo_data(out=None):
    """Write ../data/ch13-fund.json and ../data/ch13-khan.json for the three demonstrations."""
    import base64
    os.makedirs(DATA_OUT, exist_ok=True)
    F = fund()
    t, p = t_one_sample(F)
    tab = fund_table()
    J = {"m": int(F.shape[1]), "n": int(F.shape[0]), "t": np.round(t, 10).tolist(), "p": [float("%.10g" % v) for v in p],
         "mini": np.round(F.iloc[:, :5].to_numpy().T, 6).tolist(),
         "ref": {"bh": {str(q): int(bh(p, q).sum()) for q in (0.05, 0.1, 0.2, 0.3)},
                 "bonf": {str(a): int(bonferroni(p, a).sum()) for a in (0.05, 0.1, 0.3)},
                 "holm": {str(a): int(holm(p, a).sum()) for a in (0.05, 0.1, 0.3)},
                 "mini_p": tab["p"].tolist(), "q10": p_bh(p)[:10].tolist()}}
    with open(os.path.join(DATA_OUT, "ch13-fund.json"), "w") as f:
        json.dump(J, f, separators=(",", ":"))
    x1, x2 = khan_pair()
    Z = np.vstack([x1, x2]).astype(np.float32)          # 54 x 2308, gene-major below
    T, pt = t_two_sample(x1, x2)
    P = khan_permutations()
    R, fdr = plugin_fdr(P["c"], P["N"], int(P["B"]))
    K = {"n1": int(len(x1)), "n2": int(len(x2)), "m": int(Z.shape[1]),
         "x": base64.b64encode(np.ascontiguousarray(Z.T).tobytes()).decode("ascii"),
         "permp": np.round(P["Nj"] / int(P["B"]), 4).tolist(),
         "ref": {"T11": float(T[10]), "p11": float(pt[10]), "T877": float(T[876]), "p877": float(pt[876]),
                 "perm11": float(np.mean(np.abs(P["T11"]) >= abs(T[10]))),
                 "perm877": float(np.mean(np.abs(P["T877"]) >= abs(T[876]))),
                 "fdr500_bh": float(bh_estimated_fdr(pt)[499]), "fdr500_perm": float(fdr[499]),
                 "R_bh_01": int(bh(pt, 0.1).sum()), "R_perm_01": int(R[fdr <= 0.1].max()),
                 "T1955": float(T[1954]), "p1955": float(pt[1954]), "T1408": float(T[1407]), "p1408": float(pt[1407]),
                 "perm1408": float(P["Nj"][1407] / int(P["B"])),
                 "B": int(P["B"])}}
    with open(os.path.join(DATA_OUT, "ch13-khan.json"), "w") as f:
        json.dump(K, f, separators=(",", ":"))
    for name in ("ch13-fund.json", "ch13-khan.json"):
        print("wrote", name, "%.0f KB" % (os.path.getsize(os.path.join(DATA_OUT, name)) / 1024))


def report(out=None):
    """Print the numbers quoted in the notes."""
    print(fund_table().round(4))
    F = fund()
    p = t_one_sample(F)[1]
    print("Fund BH counts", {q: int(bh(p, q).sum()) for q in (0.05, 0.1, 0.3)}, "Bonferroni",
          {a: int(bonferroni(p, a).sum()) for a in (0.05, 0.1, 0.3)}, "p < 0.001:", int((p < 0.001).sum()))
    P = khan_permutations()
    R, fdr = plugin_fdr(P["c"], P["N"], int(P["B"]))
    print("Khan gene 11 T %.4f p %.4f perm %.4f; gene 877 T %.4f p %.4f perm %.4f" % (
        P["T"][10], P["p"][10], np.mean(np.abs(P["T11"]) >= abs(P["T"][10])),
        P["T"][876], P["p"][876], np.mean(np.abs(P["T877"]) >= abs(P["T"][876]))))
    print("FDR at 500 rejections: BH %.4f, BH q-value %.4f, plug-in %.4f" % (
        bh_estimated_fdr(P["p"])[499], np.sort(p_bh(P["p"]))[499], fdr[499]))
    R100, f100 = plugin_fdr(P["c100"], P["N100"], int(P["B"]))
    print("100 genes: max R with FDR <= 0.1: %d, <= 0.2: %d" % (R100[f100 <= 0.1].max(), R100[f100 <= 0.2].max()))


FIGURES = {"13.1": fig_13_1, "13.2": fig_13_2, "13.3": fig_13_3, "13.4": fig_13_4, "13.5": fig_13_5,
           "13.6": fig_13_6, "13.7": fig_13_7, "13.8": fig_13_8, "13.9": fig_13_9, "13.10": fig_13_10,
           "13.11": fig_13_11, "data": export_demo_data, "report": report}

if __name__ == "__main__":
    common.run(FIGURES, CH)
