"""Wasserman (2004), Chapter 15: Inference About Independence.

Companion code for Chapter 15 of L. Wasserman, All of Statistics: A Concise Course in Statistical
Inference (Springer, 2004), pp. 239-248, written from scratch with numpy, scipy and matplotlib. The
functions are short and meant to be read next to the lecture notes (../ch15-independence.html):

* Section 15.1, two binary variables: the margins and expected counts of a table (expected), the
  likelihood ratio statistic (15.4)/(15.12) and Pearson's statistic (15.5)/(15.13) (lrt_stat,
  pearson_stat, independence_tests), the odds ratio and log odds ratio with their delta-method
  standard errors (15.6)-(15.8) and the half-corrected estimator (15.9) (odds_ratio, log_or_se),
  the Wald test and the two confidence intervals for the odds ratio (or_intervals); the exact
  sampling distribution of every 2 x 2 statistic, found by listing all multinomial tables
  (enumerate_2x2), which gives exact sizes, coverages and biases (exact_size_2x2,
  exact_coverage_2x2, exact_bias_2x2) without simulation.
* Section 15.2, two discrete variables: the same statistics for an I x J table, standardised and
  adjusted residuals, and the null distribution of the statistics by simulation, either from the
  multinomial with the fitted independence probabilities or by permuting one variable (which
  keeps both margins fixed): null_sim, with the seeded generator of the browser demonstrations.
* Section 15.3, two continuous variables: the sample correlation, its t test, Fisher's
  z interval, a permutation test, and Spearman's and Kendall's rank correlations (implemented
  here and compared with scipy), and the rejection rate of the z test of zero correlation for
  independent and for uncorrelated-but-dependent pairs (corr_test_size).
* Section 15.4, one continuous and one discrete variable: the two-sample Kolmogorov-Smirnov
  statistic (ks_stat, in integer arithmetic so that ties are handled exactly), its exact null
  distribution for continuous data by a lattice-path recursion (ks_exact_sf), the closed form of
  Gnedenko and Korolyuk for equal sample sizes (gk_sf), Kolmogorov's limit H(t) of (15.14) and its
  inverse (kolmogorov_cdf, kolmogorov_isf), and the permutation p-value, which is exact with ties
  (ks_perm_exact by listing all splits, ks_perm_mc with the demonstrations' generator).
* Section 15.5, appendix: odds, relative risk and the ratio psi / xi of Theorem 15.13 (rr_or),
  and the three sampling designs (multinomial, prospective, case-control) simulated from one
  population (design_sim).
* Exercises: the death-sentencing table (ex4), Age against Financial status in the Montana poll
  (ex5, montana.dat), January temperature against latitude (ex6, temp.dat), and calcium against the
  drop in blood pressure (ex7, calcium.dat).
* js_uniforms / js_normals / JSRandom: the seeded generator DK.rng of courses/demo-kit.js, so that
  the Monte Carlo numbers on the demonstration pages can be reproduced here exactly.

The chapter has no figures of its own (its only displays are tables), so nothing of the book is
reproduced as a figure and nothing is skipped. Five supplementary figures, numbered 15.S1 to 15.S5,
illustrate the gaps filled in the notes:
  15.S1 two binary variables: exact size of the three tests of independence, exact coverage of the
        intervals for the odds ratio, and the bias of the log odds ratio with and without the 1/2;
  15.S2 I x J tables: standardised residuals for Example 15.10 and Exercise 15.5, and the null
        distribution of the two statistics for a sparse table;
  15.S3 two continuous variables: Exercise 15.6, an uncorrelated but dependent pair, and the
        rejection rate of the test of zero correlation;
  15.S4 the two-sample Kolmogorov-Smirnov test: Exercise 15.7, the exact null distribution against
        the limit H(t), and the exact size of the asymptotic test;
  15.S5 the appendix: odds ratio against relative risk, and the three sampling designs.

Run `python3 ch15_independence.py --list`, `python3 ch15_independence.py` (all figures, about half a
minute on one CPU thread) or `python3 ch15_independence.py --fig 15.S4`. Files go to ../img/ch15/;
the data of the demonstrations go to ../data/ch15-independence.json.
"""
import itertools
import json
import math
import os

import numpy as np
from scipy import special, stats

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, savefig, plt

CH = 15
M32 = 0xFFFFFFFF
HERE = os.path.dirname(os.path.abspath(__file__))

# =========================================================================== the book's tables
# Example 15.5 (Johnson and Johnson 1972): rows tonsillectomy / none, columns Hodgkin's / none.
TONSIL = np.array([[90, 165], [84, 307]])
TONSIL_ROWS, TONSIL_COLS = ["Tonsillectomy", "No tonsillectomy"], ["Hodgkin's disease", "No disease"]
# Example 15.10 (Dunsmore et al. 1987): histological type by response to treatment.
HODGKIN = np.array([[74, 18, 12], [68, 16, 12], [154, 54, 58], [18, 10, 44]])
HODGKIN_ROWS, HODGKIN_COLS = ["LP", "NS", "MC", "LD"], ["Positive", "Partial", "None"]
# Exercise 15.4: death sentences by race of the victim (the book's approximate re-creation).
DEATH = np.array([[14, 641], [62, 594]])
DEATH_ROWS, DEATH_COLS = ["Black victim", "White victim"], ["Death sentence", "No death sentence"]


# =========================================================================== the demos' generator
def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """DK.rng(seed) of courses/demo-kit.js, one call at a time: u() is mulberry32 and norm() the
    Box-Muller transform that returns r cos(theta) and keeps r sin(theta) for the next call."""

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        x = 0.0
        while x == 0.0:
            x = self.u()
        r = math.sqrt(-2.0 * math.log(x))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


def js_uniforms(seed, n, start=0):
    """Uniforms number start+1, ..., start+n of DK.rng(seed), all at once: mulberry32 adds a constant
    to its state at each call, so the j-th output is a hash of seed + j * 0x6D2B79F5 (mod 2^32)."""
    j = np.arange(start + 1, start + n + 1, dtype=np.uint64)
    m = np.uint64(M32)
    a = (np.uint64(int(seed) & M32) + j * np.uint64(0x6D2B79F5)) & m
    t = ((a ^ (a >> np.uint64(15))) * (np.uint64(1) | a)) & m
    t = ((t + (((t ^ (t >> np.uint64(7))) * (np.uint64(61) | t)) & m)) & m) ^ t
    return ((t ^ (t >> np.uint64(14))) & m).astype(np.float64) / 4294967296.0


def js_normals(seed, n):
    """The first n values of DK.rng(seed).norm() (Box-Muller pairs: cos first, then the cached sin)."""
    g = JSRandom(seed)
    return np.array([g.norm() for _ in range(n)])


# =========================================================================== Sections 15.1-15.2: tables
def expected(X):
    """Expected counts under independence, E_ij = X_i. X_.j / n (the restricted MLE times n)."""
    X = np.asarray(X, float)
    return np.outer(X.sum(1), X.sum(0)) / X.sum()


def lrt_stat(X):
    """Likelihood ratio statistic (15.4)/(15.12): T = 2 sum X_ij log(X_ij X.. / (X_i. X_.j)), 0 log 0 = 0."""
    X = np.asarray(X, float)
    E = expected(X)
    with np.errstate(divide="ignore", invalid="ignore"):
        terms = np.where(X > 0, X * np.log(X / E), 0.0)
    return 2.0 * terms.sum()


def pearson_stat(X):
    """Pearson's statistic (15.5)/(15.13): U = sum (X_ij - E_ij)^2 / E_ij."""
    X = np.asarray(X, float)
    E = expected(X)
    return float(((X - E) ** 2 / E).sum())


def dof(X):
    X = np.asarray(X)
    return (X.shape[0] - 1) * (X.shape[1] - 1)


def independence_tests(X):
    """T, U, the degrees of freedom (I-1)(J-1) and the two chi-squared p-values."""
    T, U, k = lrt_stat(X), pearson_stat(X), dof(X)
    return dict(T=T, U=U, df=k, pT=float(stats.chi2.sf(T, k)), pU=float(stats.chi2.sf(U, k)))


def residuals(X):
    """Pearson residuals (X - E)/sqrt(E) (their squares add up to U) and adjusted residuals
    (X - E)/sqrt(E (1 - X_i./n)(1 - X_.j/n)), which are approximately N(0, 1) under independence."""
    X = np.asarray(X, float)
    n = X.sum()
    E = expected(X)
    a, b = X.sum(1) / n, X.sum(0) / n
    pr = (X - E) / np.sqrt(E)
    adj = (X - E) / np.sqrt(E * np.outer(1 - a, 1 - b))
    return pr, adj


def pearson_2x2_closed(X):
    """The closed form of Pearson's statistic for a 2 x 2 table, n (X00 X11 - X01 X10)^2 / (row and
    column totals)."""
    X = np.asarray(X, float)
    n = X.sum()
    return n * (X[0, 0] * X[1, 1] - X[0, 1] * X[1, 0]) ** 2 / (X.sum(1).prod() * X.sum(0).prod())


def two_prop_z(X):
    """Pooled two-proportion z statistic comparing P(column 0 | row 0) with P(column 0 | row 1); its
    square is Pearson's U."""
    X = np.asarray(X, float)
    n1, n2 = X[0].sum(), X[1].sum()
    p1, p2 = X[0, 0] / n1, X[1, 0] / n2
    pp = X[:, 0].sum() / X.sum()
    return (p1 - p2) / math.sqrt(pp * (1 - pp) * (1 / n1 + 1 / n2))


def odds_ratio(X, c=0.0):
    """psi-hat = X00 X11 / (X01 X10) of (15.6) (c = 0) or the corrected estimator (15.9) (c = 1/2)."""
    X = np.asarray(X, float) + c
    return X[0, 0] * X[1, 1] / (X[0, 1] * X[1, 0])


def log_or_se(X, c=0.0):
    """gamma-hat = log psi-hat and its delta-method standard error (15.7), sqrt(sum 1 / X_ij)."""
    X = np.asarray(X, float) + c
    g = math.log(X[0, 0]) + math.log(X[1, 1]) - math.log(X[0, 1]) - math.log(X[1, 0])
    return g, math.sqrt((1.0 / X).sum())


def or_intervals(X, alpha=0.05, c=0.0):
    """Everything of Theorem 15.6, Remark 15.7 and the Wald test: psi, gamma, their standard errors,
    W and its p-value, and the intervals gamma +- z se, psi +- z se(psi) and exp(gamma +- z se) (15.10)."""
    z = stats.norm.isf(alpha / 2)
    g, s = log_or_se(X, c)
    psi = math.exp(g)
    W = g / s
    return dict(psi=psi, gamma=g, se_gamma=s, se_psi=psi * s, W=W, pW=float(2 * stats.norm.sf(abs(W))),
                ci_gamma=(g - z * s, g + z * s), ci_psi_direct=(psi - z * psi * s, psi + z * psi * s),
                ci_psi_log=(math.exp(g - z * s), math.exp(g + z * s)))


# =========================================================================== exact 2 x 2 calculations
def enumerate_2x2(n):
    """All 2 x 2 tables with total n, as an (m, 4) integer array of (X00, X01, X10, X11)."""
    rows = []
    for a in range(n + 1):
        for b in range(n + 1 - a):
            c = np.arange(n + 1 - a - b)
            rows.append(np.column_stack([np.full_like(c, a), np.full_like(c, b), c, n - a - b - c]))
    return np.vstack(rows)


def multinomial_logpmf(T, n, p):
    """log P(X = t) for the rows t of T under Multinomial(n, p)."""
    p = np.asarray(p, float)
    with np.errstate(divide="ignore"):
        lp = np.log(p)
    out = special.gammaln(n + 1) - special.gammaln(T + 1).sum(1)
    out = out + np.where(T > 0, T * lp, 0.0).sum(1)
    return out


def table_probs(margins_a, margins_b, psi):
    """Cell probabilities (p00, p01, p10, p11) with row margin P(row 0) = a, column margin
    P(column 0) = b and odds ratio psi (the unique solution in (0, 1)^4; psi = 1 gives a b)."""
    a, b = margins_a, margins_b
    if abs(psi - 1) < 1e-14:
        p00 = a * b
    else:  # p00 (1 - a - b + p00) = psi (a - p00)(b - p00): a quadratic in p00
        A = 1 - psi
        B = (1 - a - b) + psi * (a + b)
        C = -psi * a * b
        r = np.roots([A, B, C])
        lo, hi = max(0.0, a + b - 1), min(a, b)
        p00 = float([x.real for x in r if lo - 1e-12 <= x.real <= hi + 1e-12][0])
    return np.array([p00, a - p00, b - p00, 1 - a - b + p00])


def _stats_rows(T, c=0.0):
    """T, U, gamma-hat and se for each row of an (m, 4) array of tables (nan where undefined)."""
    X = T.astype(float)
    n = X.sum(1)
    r0, r1 = X[:, 0] + X[:, 1], X[:, 2] + X[:, 3]
    c0, c1 = X[:, 0] + X[:, 2], X[:, 1] + X[:, 3]
    with np.errstate(divide="ignore", invalid="ignore"):
        E = np.column_stack([r0 * c0, r0 * c1, r1 * c0, r1 * c1]) / n[:, None]
        Tl = 2 * np.where(X > 0, X * np.log(X / E), 0.0).sum(1)
        U = ((X - E) ** 2 / E).sum(1)
        Y = X + c
        g = np.log(Y[:, 0]) + np.log(Y[:, 3]) - np.log(Y[:, 1]) - np.log(Y[:, 2])
        se = np.sqrt((1.0 / Y).sum(1))
    ok_margin = (r0 > 0) & (r1 > 0) & (c0 > 0) & (c1 > 0)
    ok_cells = (Y > 0).all(1)
    Tl = np.where(ok_margin, Tl, np.nan)
    U = np.where(ok_margin, U, np.nan)
    g = np.where(ok_cells, g, np.nan)
    se = np.where(ok_cells, se, np.nan)
    return Tl, U, g, se


def exact_size_2x2(n, a, b, alpha=0.05, psi=1.0):
    """Exact rejection probabilities of the likelihood ratio, Pearson and Wald (gamma = 0) tests at
    level alpha when the cell probabilities have margins a, b and odds ratio psi (psi = 1: the size at
    that point of the null). A statistic that is undefined (an empty margin, or a zero cell for W)
    counts as no rejection."""
    T = enumerate_2x2(n)
    w = np.exp(multinomial_logpmf(T, n, table_probs(a, b, psi)))
    Tl, U, g, se = _stats_rows(T)
    crit, z = stats.chi2.isf(alpha, 1), stats.norm.isf(alpha / 2)
    with np.errstate(invalid="ignore"):
        rT = np.nan_to_num(Tl) > crit
        rU = np.nan_to_num(U) > crit
        rW = np.abs(np.nan_to_num(g / se)) > z
    return float(w @ rT), float(w @ rU), float(w @ rW)


def exact_coverage_2x2(n, p, alpha=0.05):
    """Exact coverage of three intervals for psi under Multinomial(n, p): exp(gamma +- z se) (15.10),
    psi +- z se(psi), and exp(gamma~ +- z se~) with 1/2 added to every cell (15.9). Tables with a zero
    cell (where the first two are undefined) count as misses for those two. Also returns P(zero cell)."""
    p = np.asarray(p, float)
    psi = p[0] * p[3] / (p[1] * p[2])
    T = enumerate_2x2(n)
    w = np.exp(multinomial_logpmf(T, n, p))
    z = stats.norm.isf(alpha / 2)
    _, _, g, se = _stats_rows(T)
    _, _, gh, seh = _stats_rows(T, 0.5)
    lg = math.log(psi)
    with np.errstate(invalid="ignore"):
        c_log = np.abs(np.nan_to_num(g - lg, nan=np.inf)) <= z * np.nan_to_num(se, nan=0.0)
        ph = np.exp(g)
        c_dir = np.abs(np.nan_to_num(ph - psi, nan=np.inf)) <= z * np.nan_to_num(ph * se, nan=0.0)
        c_hal = np.abs(gh - lg) <= z * seh
    zero = ~(T > 0).all(1)
    return dict(log=float(w @ c_log), direct=float(w @ c_dir), haldane=float(w @ c_hal), pzero=float(w @ zero))


def exact_bias_2x2(n, p):
    """E(gamma-hat | no zero cell) - gamma and E(gamma~) - gamma (1/2 added to every cell), exactly."""
    p = np.asarray(p, float)
    lg = math.log(p[0] * p[3] / (p[1] * p[2]))
    T = enumerate_2x2(n)
    w = np.exp(multinomial_logpmf(T, n, p))
    _, _, g, _ = _stats_rows(T)
    _, _, gh, _ = _stats_rows(T, 0.5)
    ok = ~np.isnan(g)
    return float((w[ok] @ g[ok]) / w[ok].sum() - lg), float(w @ gh - lg), float(w[ok].sum())


# =========================================================================== simulated null distributions
def sample_table(u, cum, K):
    """Cell counts of len(u) observations assigned by the inverse cdf (first k with u < cum[k]),
    exactly as the demonstration does."""
    idx = np.minimum(np.searchsorted(cum, u, side="right"), K - 1)
    return np.bincount(idx, minlength=K)


def js_cumsum(p):
    """Left-to-right running sums (the order of a JavaScript loop)."""
    out, s = [], 0.0
    for v in p:
        s = s + float(v)
        out.append(s)
    return np.array(out)


def null_sim(X, B, seed, scheme="multinomial"):
    """B tables simulated under independence with the observed total n, and their statistics T and U.
    scheme = 'multinomial': X* ~ Multinomial(n, p-hat_i. p-hat_.j) (the margins vary from table to
    table; a table with an empty margin gets T = U = 0, i.e. it never counts as extreme);
    scheme = 'permutation': the column labels of the n individuals are shuffled (Fisher-Yates), which
    keeps both margins fixed and gives the exact conditional null distribution of the statistics.
    The random numbers are those of DK.rng(seed), consumed in the same order as in the demonstration."""
    X = np.asarray(X, int)
    I, J = X.shape
    n = int(X.sum())
    Ts, Us = np.empty(B), np.empty(B)
    if scheme == "multinomial":
        a, b = X.sum(1) / n, X.sum(0) / n
        p = np.array([a[i] * b[j] for i in range(I) for j in range(J)])
        cum = js_cumsum(p)
        u = js_uniforms(seed, n * B)
        for r in range(B):
            Y = sample_table(u[r * n:(r + 1) * n], cum, I * J).reshape(I, J)
            if (Y.sum(1) == 0).any() or (Y.sum(0) == 0).any():
                Ts[r] = Us[r] = 0.0
            else:
                Ts[r], Us[r] = js_lrt(Y), js_pearson(Y)
    else:
        rows = np.repeat(np.repeat(np.arange(I), J), X.ravel())
        cols0 = np.repeat(np.tile(np.arange(J), I), X.ravel())
        g = JSRandom(seed)
        for r in range(B):
            cols = cols0.copy()
            for i in range(n - 1, 0, -1):
                j = int(g.u() * (i + 1))
                cols[i], cols[j] = cols[j], cols[i]
            Y = np.zeros((I, J), int)
            np.add.at(Y, (rows, cols), 1)
            Ts[r], Us[r] = js_lrt(Y), js_pearson(Y)
    return Ts, Us


def js_lrt(Y):
    """T computed term by term in the order the demonstration uses (rows, then columns)."""
    I, J = Y.shape
    n = 0
    for i in range(I):
        for j in range(J):
            n += int(Y[i, j])
    R = [int(Y[i].sum()) for i in range(I)]
    C = [int(Y[:, j].sum()) for j in range(J)]
    s = 0.0
    for i in range(I):
        for j in range(J):
            x = int(Y[i, j])
            if x > 0:
                e = R[i] * C[j] / n
                s += x * math.log(x / e)
    return 2 * s


def js_pearson(Y):
    I, J = Y.shape
    n = int(Y.sum())
    R = [int(Y[i].sum()) for i in range(I)]
    C = [int(Y[:, j].sum()) for j in range(J)]
    s = 0.0
    for i in range(I):
        for j in range(J):
            e = R[i] * C[j] / n
            d = int(Y[i, j]) - e
            s += d * d / e
    return s


def mc_pvalue(sim, obs, tol=1e-9):
    """Monte Carlo p-value (1 + #{sim >= obs}) / (B + 1), with a relative tolerance for ties."""
    sim = np.asarray(sim)
    return (1 + int((sim >= obs - tol * max(1.0, abs(obs))).sum())) / (len(sim) + 1)


# =========================================================================== Section 15.3: correlation
def corr(x, y):
    x, y = np.asarray(x, float), np.asarray(y, float)
    xc, yc = x - x.mean(), y - y.mean()
    return float((xc * yc).sum() / math.sqrt((xc ** 2).sum() * (yc ** 2).sum()))


def corr_inference(x, y, alpha=0.05):
    """r, the t statistic r sqrt(n-2)/sqrt(1-r^2) with its exact p-value under bivariate Normal
    independence, the z test sqrt(n-3) atanh(r), Fisher's interval tanh(atanh r +- z / sqrt(n-3)),
    and the delta-method interval r +- z (1 - r^2)/sqrt(n)."""
    n = len(x)
    r = corr(x, y)
    t = r * math.sqrt(n - 2) / math.sqrt(1 - r * r)
    zf = math.atanh(r)
    zc = stats.norm.isf(alpha / 2)
    return dict(n=n, r=r, t=t, p_t=float(2 * stats.t.sf(abs(t), n - 2)), z=zf * math.sqrt(n - 3),
                p_z=float(2 * stats.norm.sf(abs(zf) * math.sqrt(n - 3))),
                ci_fisher=(math.tanh(zf - zc / math.sqrt(n - 3)), math.tanh(zf + zc / math.sqrt(n - 3))),
                ci_delta=(r - zc * (1 - r * r) / math.sqrt(n), r + zc * (1 - r * r) / math.sqrt(n)))


def ranks(x):
    """Mid-ranks (1-based), ties get the average of their positions."""
    x = np.asarray(x, float)
    o = np.argsort(x, kind="mergesort")
    r = np.empty(len(x))
    xs = x[o]
    i = 0
    while i < len(x):
        j = i
        while j + 1 < len(x) and xs[j + 1] == xs[i]:
            j += 1
        r[o[i:j + 1]] = 0.5 * (i + j) + 1
        i = j + 1
    return r


def spearman(x, y):
    """Spearman's rank correlation: the correlation of the mid-ranks."""
    return corr(ranks(x), ranks(y))


def kendall_tau_b(x, y):
    """Kendall's tau-b, (concordant - discordant) / sqrt((n0 - ties in x)(n0 - ties in y)), n0 = n(n-1)/2."""
    x, y = np.asarray(x, float), np.asarray(y, float)
    dx = np.sign(x[:, None] - x[None, :])
    dy = np.sign(y[:, None] - y[None, :])
    iu = np.triu_indices(len(x), 1)
    s = (dx * dy)[iu].sum()
    tx, ty = (dx[iu] != 0).sum(), (dy[iu] != 0).sum()
    return float(s / math.sqrt(tx * ty))


def perm_corr_pvalue(x, y, B, seed):
    """Two-sided Monte Carlo permutation p-value for |r| (numpy generator)."""
    rng = np.random.default_rng(seed)
    r0 = abs(corr(x, y))
    y = np.asarray(y, float)
    rs = np.array([abs(corr(x, rng.permutation(y))) for _ in range(B)])
    return (1 + int((rs >= r0 - 1e-12).sum())) / (B + 1)


def corr_test_size(kind, n, reps, rng, alpha=0.05):
    """Rejection rate of the z test |sqrt(n-3) atanh(r)| > z_{alpha/2} when (X, Y) are
    'normal': independent N(0,1); 'exp': independent Exponential(1); 'signflip': X ~ N(0,1),
    Y = S X with S = +-1 independent of X (uncorrelated, N(0,1) margins, dependent)."""
    X = rng.standard_normal((reps, n)) if kind != "exp" else rng.exponential(size=(reps, n))
    if kind == "signflip":
        Y = X * rng.choice([-1.0, 1.0], size=(reps, n))
    elif kind == "exp":
        Y = rng.exponential(size=(reps, n))
    else:
        Y = rng.standard_normal((reps, n))
    Xc = X - X.mean(1, keepdims=True)
    Yc = Y - Y.mean(1, keepdims=True)
    r = (Xc * Yc).sum(1) / np.sqrt((Xc ** 2).sum(1) * (Yc ** 2).sum(1))
    with np.errstate(divide="ignore"):   # r = +-1 can occur for tiny n: arctanh is infinite, the test rejects
        return float((np.abs(np.arctanh(r)) * math.sqrt(n - 3) > stats.norm.isf(alpha / 2)).mean())


# =========================================================================== Section 15.4: Kolmogorov-Smirnov
def ks_stat(x1, x2):
    """The two-sample statistic D = sup |F1-hat - F2-hat| as the integer K = D n1 n2 (exact, ties
    handled by evaluating both empirical cdfs after each distinct pooled value), and the point x
    where the supremum is first reached. Returns (K, D, x)."""
    x1, x2 = np.sort(np.asarray(x1, float)), np.sort(np.asarray(x2, float))
    n1, n2 = len(x1), len(x2)
    v = np.unique(np.concatenate([x1, x2]))
    a = np.searchsorted(x1, v, side="right")
    b = np.searchsorted(x2, v, side="right")
    d = np.abs(a * n2 - b * n1)
    k = int(np.argmax(d))
    return int(d[k]), d[k] / (n1 * n2), float(v[k])


def ks_exact_sf(n1, n2, K):
    """P(D n1 n2 >= K) under H0 for continuous data: the pooled ordering is a uniformly random
    lattice path from (0, 0) to (n1, n2), and D n1 n2 = max |i n2 - j n1| over the path. The
    recursion carries the probability of reaching (i, j) without touching the band edge
    |i n2 - j n1| >= K and adds up the probability of the first touch."""
    if K <= 0:
        return 1.0
    P = [[0.0] * (n2 + 1) for _ in range(n1 + 1)]
    P[0][0] = 1.0
    hit = 0.0
    for i in range(n1 + 1):
        for j in range(n2 + 1):
            if i == 0 and j == 0:
                continue
            m = 0.0
            if i > 0:
                r1, r2 = n1 - i + 1, n2 - j
                m += P[i - 1][j] * (r1 / (r1 + r2))
            if j > 0:
                r1, r2 = n1 - i, n2 - j + 1
                m += P[i][j - 1] * (r2 / (r1 + r2))
            if abs(i * n2 - j * n1) >= K:
                hit += m
            else:
                P[i][j] = m
    return min(1.0, hit)


def gk_sf(n, k):
    """Gnedenko-Korolyuk: for n1 = n2 = n, P(D >= k/n) = 2 sum_{j>=1} (-1)^{j-1} C(2n, n-jk) / C(2n, n)."""
    if k <= 0:
        return 1.0
    tot = 0
    j = 1
    while n - j * k >= 0:
        tot += (-1) ** (j - 1) * math.comb(2 * n, n - j * k)
        j += 1
    return 2 * tot / math.comb(2 * n, n)


def kolmogorov_cdf(t):
    """H(t) of (15.14): 1 - 2 sum (-1)^{j-1} exp(-2 j^2 t^2); for t < 1 the equivalent theta-function
    form sqrt(2 pi)/t sum exp(-(2k-1)^2 pi^2 / (8 t^2)) converges much faster."""
    t = float(t)
    if t <= 0:
        return 0.0
    if t < 1.0:
        s = 0.0
        for k in range(1, 30):
            s += math.exp(-(2 * k - 1) ** 2 * math.pi ** 2 / (8 * t * t))
        return math.sqrt(2 * math.pi) / t * s
    s = 0.0
    for j in range(1, 30):
        s += (-1) ** (j - 1) * math.exp(-2 * j * j * t * t)
    return 1 - 2 * s


def kolmogorov_isf(alpha):
    """H^{-1}(1 - alpha) by bisection."""
    lo, hi = 0.1, 5.0
    for _ in range(100):
        mid = 0.5 * (lo + hi)
        if kolmogorov_cdf(mid) < 1 - alpha:
            lo = mid
        else:
            hi = mid
    return 0.5 * (lo + hi)


def ks_exact_crit(n1, n2, alpha):
    """Smallest K with P(D n1 n2 >= K) <= alpha under H0 (continuous data), by bisection on K."""
    lo, hi = 1, n1 * n2 + 1
    while lo < hi:
        mid = (lo + hi) // 2
        if ks_exact_sf(n1, n2, mid) <= alpha:
            hi = mid
        else:
            lo = mid + 1
    return lo


def ks_demo_numbers(x1, x2, alpha, B, seed):
    """Every number the Kolmogorov-Smirnov demonstration shows, for the same data and seed."""
    n1, n2 = len(x1), len(x2)
    K, D, at = ks_stat(x1, x2)
    sc = math.sqrt(n1 * n2 / (n1 + n2)) / (n1 * n2)
    hcrit = kolmogorov_isf(alpha)
    w = stats.ttest_ind(x1, x2, equal_var=False)
    return dict(K=K, D=D, at=at, stat=K * sc, pExact=ks_exact_sf(n1, n2, K), pAsym=1 - kolmogorov_cdf(K * sc),
                pPerm=ks_perm_mc(x1, x2, B, seed), hcrit=hcrit, kcrit=ks_exact_crit(n1, n2, alpha),
                ecrit=ks_exact_crit(n1, n2, alpha) * sc, size=ks_exact_sf(n1, n2, math.floor(hcrit / sc) + 1),
                welchT=float(w.statistic), welchP=float(w.pvalue))


def ks_sim_samples(n1, n2, shift, ratio, seed):
    """The simulated samples of the demonstration: N(0, 1) and shift + ratio N(0, 1) from DK.rng(seed)."""
    z = js_normals(seed, n1 + n2)
    return z[:n1], np.array([shift + ratio * v for v in z[n1:]])


def ks_test(x1, x2):
    """D, the scaled statistic sqrt(n1 n2/(n1+n2)) D, the exact p-value for continuous data, and the
    asymptotic p-value 1 - H(.)."""
    n1, n2 = len(x1), len(x2)
    K, D, x = ks_stat(x1, x2)
    s = math.sqrt(n1 * n2 / (n1 + n2)) * D
    return dict(n1=n1, n2=n2, K=K, D=D, at=x, stat=s, p_exact=ks_exact_sf(n1, n2, K), p_asym=1 - kolmogorov_cdf(s))


def ks_perm_mc(x1, x2, B, seed):
    """Monte Carlo permutation p-value (1 + #{K* >= K}) / (B + 1): the pooled sample (sample 1 first)
    is shuffled by Fisher-Yates with DK.rng(seed) and the first n1 values form sample 1."""
    x1, x2 = np.asarray(x1, float), np.asarray(x2, float)
    n1 = len(x1)
    z0 = np.concatenate([x1, x2])
    K0 = ks_stat(x1, x2)[0]
    g = JSRandom(seed)
    n = len(z0)
    cnt = 0
    for _ in range(B):
        z = z0.copy()
        for i in range(n - 1, 0, -1):
            j = int(g.u() * (i + 1))
            z[i], z[j] = z[j], z[i]
        if ks_stat(z[:n1], z[n1:])[0] >= K0:
            cnt += 1
    return (1 + cnt) / (B + 1)


def ks_perm_exact(x1, x2):
    """The exact permutation p-value P(K* >= K) over all C(n, n1) splits of the pooled sample (exact
    with ties). Feasible for small samples (Exercise 15.7: C(21, 10) = 352,716 splits)."""
    x1, x2 = np.asarray(x1, float), np.asarray(x2, float)
    n1, n2 = len(x1), len(x2)
    z = np.concatenate([x1, x2])
    o = np.argsort(z, kind="mergesort")
    zs = z[o]
    last = np.r_[zs[1:] != zs[:-1], True]          # positions where a run of tied values ends
    K0 = ks_stat(x1, x2)[0]
    n = n1 + n2
    combs = np.array(list(itertools.combinations(range(n), n1)), dtype=np.int32)
    M = np.zeros((len(combs), n), dtype=np.int16)
    M[np.arange(len(combs))[:, None], combs] = 1     # 1 = this sorted position belongs to sample 1
    a = np.cumsum(M, axis=1)[:, last]
    b = (np.arange(1, n + 1)[None, :] - np.cumsum(M, axis=1))[:, last]
    K = np.abs(a * n2 - b * n1).max(1)
    return float((K >= K0).mean()), len(combs)


# =========================================================================== Section 15.5: appendix
def rr_or(pD_E, pD_Ec):
    """Odds ratio psi, relative risk xi and the ratio psi / xi = (1 - P(D|Ec)) / (1 - P(D|E))."""
    o1, o0 = pD_E / (1 - pD_E), pD_Ec / (1 - pD_Ec)
    psi, xi = o1 / o0, pD_E / pD_Ec
    return psi, xi, psi / xi


def design_sim(pE, pD_Ec, psi, n_multi, n_exposed, n_unexposed, n_cases, n_controls, reps, rng):
    """Corrected log odds ratios (1/2 in every cell) from three designs drawn from one population:
    a multinomial sample of n_multi people, a prospective sample of fixed exposure groups, and a
    case-control sample of fixed numbers of cases and controls. Returns the three arrays and the
    true log odds ratio."""
    o1 = psi * pD_Ec / (1 - pD_Ec)
    pD_E = o1 / (1 + o1)
    # cells in the appendix layout: rows E^c, E; columns D^c, D
    p = np.array([(1 - pE) * (1 - pD_Ec), (1 - pE) * pD_Ec, pE * (1 - pD_E), pE * pD_E])
    M = rng.multinomial(n_multi, p, size=reps).astype(float)
    x01 = rng.binomial(n_unexposed, pD_Ec, size=reps)
    x11 = rng.binomial(n_exposed, pD_E, size=reps)
    Pr = np.column_stack([n_unexposed - x01, x01, n_exposed - x11, x11]).astype(float)
    pD = p[1] + p[3]
    pE_D, pE_Dc = p[3] / pD, p[2] / (1 - pD)
    x11c = rng.binomial(n_cases, pE_D, size=reps)
    x10c = rng.binomial(n_controls, pE_Dc, size=reps)
    Cc = np.column_stack([n_controls - x10c, n_cases - x11c, x10c, x11c]).astype(float)

    def lg(T):
        T = T + 0.5
        return np.log(T[:, 0]) + np.log(T[:, 3]) - np.log(T[:, 1]) - np.log(T[:, 2])
    return lg(M), lg(Pr), lg(Cc), math.log(psi), p


# =========================================================================== the book's data files
def _data_lines(name, header_start):
    lines = open(common.aos_file(name), encoding="latin-1").read().splitlines()
    k = next(i for i, s in enumerate(lines) if s.startswith(header_start))
    return lines[k + 1:]


def montana():
    """montana.dat: (AGE, FIN) for the respondents with both recorded (AGE 1 <35, 2 35-54, 3 55+;
    FIN 1 worse, 2 same, 3 better than a year ago)."""
    out = []
    for s in _data_lines("montana.dat", "AGE"):
        t = s.split()
        if len(t) == 7 and t[0] != "*" and t[5] != "*":
            out.append((int(t[0]), int(t[5])))
    return np.array(out)


def montana_table():
    A = montana()
    X = np.zeros((3, 3), int)
    for a, f in A:
        X[a - 1, f - 1] += 1
    return X


def temperatures():
    """temp.dat: January minimum temperature (F) and latitude of 56 US cities."""
    rows = []
    for s in _data_lines("temp.dat", "City"):
        t = s.split("\t")
        if len(t) >= 4 and t[1].strip():
            rows.append((t[0].strip(), float(t[1]), float(t[2]), float(t[3])))
    return rows


def calcium():
    """calcium.dat: drop in systolic blood pressure (Begin - End) for the calcium and placebo groups."""
    ca, pl = [], []
    for s in _data_lines("calcium.dat", "Treatment"):
        t = s.split()
        if len(t) == 4:
            (ca if t[0] == "Calcium" else pl).append(float(t[3]))
    return np.array(ca), np.array(pl)


def coris():
    """coris.dat (South African heart disease data, 462 men): a dict of columns."""
    lines = open(common.aos_file("coris.dat")).read().splitlines()
    head = lines[0].split(",")
    rows = [[float(v) for v in s.split(",")] for s in lines[1:] if s[:1].isdigit()]
    A = np.array(rows)
    return {h: A[:, i] for i, h in enumerate(head)}


# =========================================================================== exercises
def ex4():
    return dict(tests=independence_tests(DEATH), odds=or_intervals(DEATH),
                rates=(DEATH[0, 0] / DEATH[0].sum(), DEATH[1, 0] / DEATH[1].sum()))


def ex5():
    X = montana_table()
    pr, adj = residuals(X)
    return dict(X=X, tests=independence_tests(X), E=expected(X), adj=adj)


def ex6(B=10000, seed=6):
    rows = temperatures()
    temp = np.array([r[1] for r in rows])
    lat = np.array([r[2] for r in rows])
    out = corr_inference(lat, temp)
    out["spearman"] = spearman(lat, temp)
    out["kendall"] = kendall_tau_b(lat, temp)
    out["p_perm"] = perm_corr_pvalue(lat, temp, B, seed)
    rng = np.random.default_rng(seed)
    bs = []
    n = len(lat)
    for _ in range(2000):
        i = rng.integers(0, n, n)
        bs.append(corr(lat[i], temp[i]))
    out["boot_se"] = float(np.std(bs, ddof=1))
    out["boot_pct"] = tuple(np.percentile(bs, [2.5, 97.5]))
    return out


def ex7():
    ca, pl = calcium()
    out = ks_test(ca, pl)
    out["p_perm_exact"], out["splits"] = ks_perm_exact(ca, pl)
    out["welch"] = stats.ttest_ind(ca, pl, equal_var=False)
    out["means"] = (ca.mean(), pl.mean())
    return out


# =========================================================================== figures
def _heat(ax, R, rows, cols, X, title, vmax=6):
    im = ax.imshow(R, cmap="RdBu_r", vmin=-vmax, vmax=vmax, aspect="auto")
    for i in range(R.shape[0]):
        for j in range(R.shape[1]):
            ax.text(j, i, "%d\n(%+.1f)" % (X[i, j], R[i, j]), ha="center", va="center", fontsize=8,
                    color="white" if abs(R[i, j]) > 0.6 * vmax else "black")
    ax.set_xticks(range(len(cols)))
    ax.set_xticklabels(cols, fontsize=8)
    ax.set_yticks(range(len(rows)))
    ax.set_yticklabels(rows, fontsize=8)
    ax.set_title(title)
    return im


def fig_15_s1(out_dir=None):
    """Figure 15.S1: exact size of the tests, exact coverage of the intervals for psi, and bias of gamma."""
    fig, axes = common.panel_grid(1, 3, size=(3.4, 2.75))
    ns = list(range(10, 151, 5))
    S = np.array([exact_size_2x2(n, 0.4, 0.27) for n in ns])
    ax = axes[0, 0]
    ax.plot(ns, S[:, 0], color=RED, label="likelihood ratio $T$")
    ax.plot(ns, S[:, 1], color=BLUE, label="Pearson $U$")
    ax.plot(ns, S[:, 2], color=GREEN, label=r"Wald $W$ for $\gamma=0$")
    ax.axhline(0.05, color="k", lw=0.8, ls=":")
    ax.set_xlabel("n")
    ax.set_ylabel("exact size")
    ax.set_title("Size of the 5% tests (margins 0.4, 0.27)")
    ax.set_ylim(0, 0.1)
    ax.legend(fontsize=7.5, loc="lower right", frameon=False)
    common.panel_label(ax, "a")
    p = TONSIL.ravel() / TONSIL.sum()
    C = [exact_coverage_2x2(n, p) for n in ns]
    ax = axes[0, 1]
    ax.plot(ns, [c["log"] for c in C], color=BLUE, label=r"$\exp(\hat\gamma\pm z\,\widehat{se})$ (15.10)")
    ax.plot(ns, [c["direct"] for c in C], color=ORANGE, label=r"$\hat\psi\pm z\,\widehat{se}(\hat\psi)$")
    ax.plot(ns, [c["haldane"] for c in C], color=PURPLE, ls="--", label="(15.10) with 1/2 added")
    ax.axhline(0.95, color="k", lw=0.8, ls=":")
    ax.set_xlabel("n")
    ax.set_ylabel("exact coverage")
    ax.set_title(r"95% intervals for $\psi$ (Example 15.5 cells)")
    ax.set_ylim(0.70, 1.0)
    ax.legend(fontsize=7.5, loc="lower right", frameon=False)
    common.panel_label(ax, "b")
    nb = list(range(20, 201, 10))
    Bb = np.array([exact_bias_2x2(n, p)[:2] for n in nb])
    ax = axes[0, 2]
    ax.plot(nb, Bb[:, 0], color=RED, label=r"$\hat\gamma$, given no zero cell")
    ax.plot(nb, Bb[:, 1], color=PURPLE, ls="--", label=r"$\tilde\gamma$ (1/2 added)")
    ax.axhline(0, color="k", lw=0.8, ls=":")
    ax.set_xlabel("n")
    ax.set_ylabel(r"exact bias $E(\cdot)-\gamma$")
    ax.set_title("Bias of the log odds ratio")
    ax.legend(fontsize=7.5, loc="upper right", frameon=False)
    common.panel_label(ax, "c", loc="lower left")
    fig.tight_layout()
    return savefig(fig, CH, "fig15-S1", out_dir)


def fig_15_s2(out_dir=None):
    """Figure 15.S2: standardised residuals (Example 15.10, Exercise 15.5) and a sparse-table null."""
    fig, axes = common.panel_grid(1, 3, size=(3.4, 2.9))
    _, adj = residuals(HODGKIN)
    _heat(axes[0, 0], adj, HODGKIN_ROWS, HODGKIN_COLS, HODGKIN, "Example 15.10: count (adj. residual)")
    axes[0, 0].set_xlabel("response")
    axes[0, 0].set_ylabel("histological type")
    common.panel_label(axes[0, 0], "a", loc="lower left")
    X5 = montana_table()
    _, adj5 = residuals(X5)
    _heat(axes[0, 1], adj5, ["<35", "35-54", "55+"], ["worse", "same", "better"], X5, "Exercise 15.5: Montana poll", vmax=4)
    axes[0, 1].set_xlabel("financial status vs a year ago")
    axes[0, 1].set_ylabel("age")
    common.panel_label(axes[0, 1], "b", loc="lower left")
    ax = axes[0, 2]
    SP = np.array([[3, 1, 2], [1, 4, 2], [2, 2, 8]])
    Ts, Us = null_sim(SP, 20000, 2024, "permutation")
    v = np.linspace(0, 16, 400)
    for s, col, lab in ((Us, BLUE, "Pearson U"), (Ts, RED, "likelihood ratio T")):
        xs = np.sort(s)
        ax.step(xs, 1 - np.arange(1, len(xs) + 1) / len(xs), color=col, where="post", label=lab)
    ax.plot(v, stats.chi2.sf(v, 4), color="k", lw=1.2, ls="--", label=r"$\chi^2_4$")
    ax.axhline(0.05, color=GREY, lw=0.6, ls=":")
    ax.set_yscale("log")
    ax.set_ylim(1e-3, 1)
    ax.set_xlim(0, 16)
    ax.set_xlabel("t")
    ax.set_ylabel("P(statistic > t), margins fixed")
    ax.set_title("A sparse 3 x 3 table, n = 25")
    ax.legend(fontsize=7.5, loc="lower left", frameon=False)
    common.panel_label(ax, "c", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig15-S2", out_dir)


def fig_15_s3(out_dir=None):
    """Figure 15.S3: Exercise 15.6, an uncorrelated dependent pair, and the size of the test of rho = 0."""
    fig, axes = common.panel_grid(1, 3, size=(3.4, 2.8))
    rows = temperatures()
    temp = np.array([r[1] for r in rows])
    lat = np.array([r[2] for r in rows])
    ci = corr_inference(lat, temp)
    ax = axes[0, 0]
    ax.plot(lat, temp, "o", ms=3.5, color=BLUE, mfc="none")
    bb = np.polyfit(lat, temp, 1)
    xx = np.linspace(lat.min(), lat.max(), 10)
    ax.plot(xx, np.polyval(bb, xx), color=RED, lw=1.2)
    ax.set_xlabel("latitude (degrees north)")
    ax.set_ylabel("January minimum (F)")
    ax.set_title("Exercise 15.6: r = %.3f, 95%% CI (%.2f, %.2f)" % (ci["r"], *ci["ci_fisher"]), fontsize=9)
    common.panel_label(ax, "a", loc="lower left")
    rng = np.random.default_rng(153)
    x = rng.standard_normal(400)
    y = x * rng.choice([-1.0, 1.0], 400)
    ax = axes[0, 1]
    ax.plot(x, y, ".", ms=3, color=PURPLE)
    ax.set_aspect("equal")
    ax.set_xlabel("Z")
    ax.set_ylabel("Y = S Z")
    ax.set_title(r"N(0,1) margins, $\rho=0$, dependent", fontsize=9.5)
    common.panel_label(ax, "b", loc="upper left")
    ax = axes[0, 2]
    ns = [10, 20, 50, 100, 200, 500]
    rng = np.random.default_rng(1503)
    for kind, col, lab in (("normal", BLUE, "independent Normals"), ("exp", GREEN, "independent Exponentials"),
                           ("signflip", PURPLE, "Y = S Z (uncorrelated)")):
        ax.plot(ns, [corr_test_size(kind, n, 4000, rng) for n in ns], "o-", color=col, ms=3.5, label=lab)
    ax.axhline(0.05, color="k", lw=0.8, ls=":")
    ax.axhline(2 * stats.norm.sf(1.96 / math.sqrt(3)), color=PURPLE, lw=0.8, ls="--")
    ax.set_xscale("log")
    ax.set_ylim(0, 0.35)
    ax.set_xlabel("n")
    ax.set_ylabel("P(reject $\\rho=0$) at 5%")
    ax.set_title("Fisher z test of zero correlation", fontsize=9.5)
    ax.legend(fontsize=7.5, loc="center right", frameon=False)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig15-S3", out_dir)


def fig_15_s4(out_dir=None):
    """Figure 15.S4: Exercise 15.7 ECDFs, the exact null distribution against H, and the exact size."""
    fig, axes = common.panel_grid(1, 3, size=(3.4, 2.8))
    ca, pl = calcium()
    r = ks_test(ca, pl)
    ax = axes[0, 0]
    for x, col, lab in ((ca, RED, "calcium (n = 10)"), (pl, BLUE, "placebo (n = 11)")):
        xs = np.sort(x)
        ax.step(np.r_[-14, xs, 20], np.r_[0, np.arange(1, len(xs) + 1) / len(xs), 1], where="post", color=col, label=lab)
    F1 = np.searchsorted(np.sort(ca), r["at"], side="right") / len(ca)
    F2 = np.searchsorted(np.sort(pl), r["at"], side="right") / len(pl)
    ax.plot([r["at"], r["at"]], [F1, F2], color="k", lw=2.2)
    ax.text(r["at"] - 0.8, 0.5 * (F1 + F2), "D = %.3f" % r["D"], fontsize=8.5, va="center", ha="right",
            bbox=dict(boxstyle="round,pad=0.15", fc="white", ec="none", alpha=0.9))
    ax.set_xlabel("drop in systolic blood pressure")
    ax.set_ylabel("empirical cdf")
    ax.set_title("Exercise 15.7", fontsize=9.5)
    ax.legend(fontsize=7.5, loc="lower right", frameon=False)
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    t = np.linspace(0.01, 2.2, 300)
    ax.plot(t, [kolmogorov_cdf(s) for s in t], color="k", lw=1.6, label="limit H(t)")
    for (n1, n2), col in (((5, 5), ORANGE), ((10, 11), RED), ((50, 50), BLUE)):
        Ks = sorted(set(abs(i * n2 - j * n1) for i in range(n1 + 1) for j in range(n2 + 1)))
        Ks = [k for k in Ks if k > 0]
        sc = math.sqrt(n1 * n2 / (n1 + n2)) / (n1 * n2)
        xs = [0.0] + [k * sc for k in Ks] + [2.2]
        F = [0.0] + [1 - ks_exact_sf(n1, n2, k + 1) for k in Ks] + [1.0]
        ax.step(xs, F, where="post", color=col, lw=1.1, label="exact, n = (%d, %d)" % (n1, n2))
    ax.set_xlim(0, 2.2)
    ax.set_xlabel(r"t")
    ax.set_ylabel(r"P$(\sqrt{n_1n_2/(n_1+n_2)}\,D\leq t)$")
    ax.set_title("Null distribution, continuous data", fontsize=9.5)
    ax.legend(fontsize=7.5, loc="lower right", frameon=False)
    common.panel_label(ax, "b", loc="upper left")
    ax = axes[0, 2]
    ns = list(range(4, 121))
    crit = kolmogorov_isf(0.05)
    for ratio, col, lab in ((1, RED, r"$n_2=n_1$"), (2, GREEN, r"$n_2=2n_1$")):
        size = []
        for n in ns:
            n1, n2 = n, ratio * n
            sc = math.sqrt(n1 * n2 / (n1 + n2)) / (n1 * n2)
            size.append(ks_exact_sf(n1, n2, math.floor(crit / sc) + 1))   # reject when K sc > crit
        ax.plot(ns, size, color=col, lw=1.1, label="asymptotic test, " + lab)
    ax.axhline(0.05, color="k", lw=0.8, ls=":")
    ax.set_xlabel(r"$n_1$")
    ax.set_ylabel("exact size")
    ax.set_ylim(0, 0.065)
    ax.set_title("Exact size of the asymptotic 5% test", fontsize=9.5)
    ax.legend(fontsize=7.5, loc="lower right", frameon=False)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig15-S4", out_dir)


def fig_15_s5(out_dir=None):
    """Figure 15.S5: odds ratio against relative risk (Theorem 15.13) and the three sampling designs."""
    fig, axes = common.panel_grid(1, 2, size=(3.9, 2.8))
    ax = axes[0, 0]
    pD = np.logspace(-4, np.log10(0.6), 300)
    pE = 0.3
    for psi, col in ((1.5, BLUE), (2.0, GREEN), (4.0, ORANGE), (10.0, RED)):
        ratio = []
        for q in pD:   # solve P(D) = pE P(D|E) + (1 - pE) P(D|Ec) for P(D|Ec) by bisection
            lo, hi = 1e-12, 1 - 1e-12
            for _ in range(80):
                m0 = 0.5 * (lo + hi)
                o1 = psi * m0 / (1 - m0)
                tot = pE * o1 / (1 + o1) + (1 - pE) * m0
                lo, hi = (m0, hi) if tot < q else (lo, m0)
            m0 = 0.5 * (lo + hi)
            o1 = psi * m0 / (1 - m0)
            ratio.append(rr_or(o1 / (1 + o1), m0)[2])
        ax.plot(pD, ratio, color=col, label=r"$\psi$ = %g" % psi)
    ax.set_xscale("log")
    ax.axhline(1, color="k", lw=0.8, ls=":")
    ax.set_xlabel("P(D), the disease prevalence")
    ax.set_ylabel(r"$\psi/\xi$ (odds ratio / relative risk)")
    ax.set_title("Theorem 15.13, P(E) = 0.3", fontsize=9.5)
    ax.set_ylim(0.8, 6.2)
    ax.legend(fontsize=7.5, loc="upper left", frameon=False)
    common.panel_label(ax, "a", loc="upper right")
    ax = axes[0, 1]
    rng = np.random.default_rng(1513)
    M, Pr, Cc, lg, _ = design_sim(0.4, 0.01, 2.0, 646, 323, 323, 174, 472, 20000, rng)
    bins = np.linspace(-3, 4.5, 76)
    for v, col, lab in ((M, GREY, "multinomial, n = 646"), (Pr, GREEN, "prospective, 323 + 323"),
                        (Cc, BLUE, "case-control, 174 + 472")):
        ax.hist(v, bins=bins, density=True, histtype="step", color=col, lw=1.4, label=lab + " (sd %.2f)" % v.std())
    ax.axvline(lg, color=RED, lw=1.2)
    ax.set_xlabel(r"$\tilde\gamma$ (1/2 added to every cell)")
    ax.set_ylabel("density")
    ax.set_title(r"Rare disease: P(D|E$^c$) = 0.01, $\psi$ = 2", fontsize=9.5)
    ax.set_ylim(0, 2.9)
    ax.legend(fontsize=7.2, loc="upper right", frameon=False)
    common.panel_label(ax, "b", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig15-S5", out_dir)


# =========================================================================== demo data
def export_json(path=None):
    """Write ../data/ch15-independence.json: the book's tables and the data of Exercises 15.5-15.7
    and of the coris variables used in the Kolmogorov-Smirnov demonstration."""
    path = path or os.path.normpath(os.path.join(HERE, "..", "data", "ch15-independence.json"))
    ca, pl = calcium()
    co = coris()
    chd = co["chd"] == 1
    d = {"note": "Data of Wasserman (2004), Chapter 15: the tables of Examples 15.5 and 15.10 and Exercise 15.4 "
                 "(printed in the book), Age by Financial status from montana.dat (Exercise 15.5), the drop in blood "
                 "pressure from calcium.dat (Exercise 15.7), and three variables of coris.dat split by coronary heart "
                 "disease. Source: https://www.stat.cmu.edu/~larry/all-of-statistics/",
         "tables": {
             "tonsil": {"X": TONSIL.tolist(), "rows": TONSIL_ROWS, "cols": TONSIL_COLS},
             "hodgkin": {"X": HODGKIN.tolist(), "rows": HODGKIN_ROWS, "cols": HODGKIN_COLS},
             "death": {"X": DEATH.tolist(), "rows": DEATH_ROWS, "cols": DEATH_COLS},
             "montana": {"X": montana_table().tolist(), "rows": ["Age < 35", "35-54", "55 and over"],
                         "cols": ["Worse", "Same", "Better"]}},
         "calcium": {"calcium": ca.tolist(), "placebo": pl.tolist()},
         "coris": {v: {"chd": co[v][chd].tolist(), "nochd": co[v][~chd].tolist()} for v in ("age", "sbp", "ldl")}}
    with open(path, "w") as f:
        json.dump(d, f, separators=(",", ":"))
    print("wrote", os.path.relpath(path, os.getcwd()))
    return path


FIGURES = {"15.S1": fig_15_s1, "15.S2": fig_15_s2, "15.S3": fig_15_s3, "15.S4": fig_15_s4, "15.S5": fig_15_s5}

if __name__ == "__main__":
    export_json()
    common.run(FIGURES, CH)
