"""Wasserman (2004), Chapter 8: The Bootstrap.

Companion code for Chapter 8 of L. Wasserman, All of Statistics: A Concise Course in Statistical
Inference (Springer, 2004), pp. 107-118, written from scratch with numpy, scipy and matplotlib. The
functions are short and meant to be read next to the lecture notes (../ch08-bootstrap.html):

* Data of the chapter: nerve() (the nerve pulse data, Example 8.2), LSAT / GPA (the law school data
  of Example 8.6, with the GPA of the last school corrected to 2.96 as in the author's errata),
  PATCH (the hormone patch data of Example 8.7) and coris_ldl() (LDL cholesterol by heart disease
  status from coris.dat, used in place of the plasma cholesterol data of Example 8.5, which are not
  on the book's website).
* Plug-in statistics: skewness, corr_rows, ratio_of_means, iqr_scale (the (q.75 - q.25) / 1.34 of
  Exercise 8.3), q7 (sample quantiles by linear interpolation, the rule used everywhere here).
* Section 8.2, bootstrap variance estimation: bootstrap() (Steps 1-3 of the book's box), v_boot()
  (equation (8.1), divisor B), boot_median_exact() (the ideal bootstrap distribution of the median,
  which needs no simulation), mc_rel_sd() (the Monte Carlo error of the bootstrap se for a given B),
  se_influence_skew() (the nonparametric delta method for the skewness, through its influence function).
* Section 8.3, bootstrap confidence intervals: boot_intervals() (Normal (8.2), pivotal (8.6) and
  percentile) and coverage_study() (the coverage experiments of Exercises 8.2 and 8.3).
* Appendix 8.5.1, the jackknife: jackknife() (v_jack and the leave-one-out values) and the exact
  variance of the median of exponential data (median_var_exp), used to show that v_jack is not
  consistent for the median.
* Exercises: n_distinct_resamples (8.4), boot_mean_moments (8.5), vboot_mean_square (8.8: the exact
  ideal bootstrap variance of the squared mean, next to the formula printed in the book).
* JSRandom-compatible generators js_uniforms / js_normals (DK.rng in courses/demo-kit.js) and the
  three demo routines cov_demo, jk_demo and breps_demo, so the numbers on the demo pages can be
  reproduced here.

Figures. Reproduced from the book: 8.1 (law school data and bootstrap correlations) and 8.2
(bootstrap replications for the patch data). The book's only other picture, the "real world /
bootstrap world" diagram of Section 8.2, is a schematic with nothing computed and is skipped.
Supplementary figures (ours), each illustrating a point filled in the notes:
  8.S1 the two approximations of the bootstrap: Monte Carlo error against B, statistical error over data sets;
  8.S2 the nerve-data skewness: bootstrap distribution, the three intervals, and the pivotal reflection;
  8.S3 coverage of the three intervals for the skewness of lognormal data (Exercise 8.2);
  8.S4 coverage and length for the scaled interquartile range of t_3 data (Exercise 8.3);
  8.S5 the jackknife against the bootstrap: consistent for smooth statistics, not for the median;
  8.S6 Exercises 8.6 and 8.7: exp(mean) of Normal data, and the maximum, where the bootstrap fails.
Run `python3 ch08_bootstrap.py --list`, `python3 ch08_bootstrap.py` (all eight figures and the demo data, about
half a minute on one CPU thread the first time; the coverage studies are cached in ./cache) or
`python3 ch08_bootstrap.py --fig 8.1`. Files go to ../img/ch08/; the demo data go to
../data/ch08-bootstrap.json.
"""
import json
import math
import os

import matplotlib.ticker
import numpy as np
from scipy import stats

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, savefig

CH = 8
M32 = 0xFFFFFFFF
HERE = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(HERE, "cache")
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data", "ch08-bootstrap.json"))
Z = {0.10: 1.6448536269514722, 0.05: 1.959963984540054, 0.01: 2.5758293035489004}


# =========================================================================== data
def nerve():
    """The 799 waiting times (seconds) between nerve pulses of the book's nerve.dat."""
    return np.array(open(common.aos_file("nerve.dat")).read().split(), dtype=float)


# Example 8.6 (Efron's law school data). The book prints 3.96 for the last GPA; the author's errata
# list corrects it to 2.96, which is the value that gives the quoted correlation 0.776.
LSAT = np.array([576, 635, 558, 578, 666, 580, 555, 661, 651, 605, 653, 575, 545, 572, 594], dtype=float)
GPA = np.array([3.39, 3.30, 2.81, 3.03, 3.44, 3.07, 3.00, 3.43, 3.36, 3.13, 3.12, 2.74, 2.76, 2.88, 2.96])

# Example 8.7 (the hormone patch data): columns placebo, old patch, new patch for eight subjects.
PATCH = np.array([[9243, 17649, 16449], [9671, 12013, 14614], [11792, 19979, 17274], [13357, 21816, 23798],
                  [9055, 13850, 12560], [6290, 9806, 10157], [12412, 17208, 16570], [18806, 29044, 26325]],
                 dtype=float)


def patch_yz():
    """Y = new - old and Z = old - placebo of Example 8.7."""
    return PATCH[:, 2] - PATCH[:, 1], PATCH[:, 1] - PATCH[:, 0]


def coris_ldl():
    """LDL cholesterol for the men without (chd = 0) and with (chd = 1) coronary heart disease in the
    book's coris.dat (South African heart disease data, 462 men): our stand-in for Example 8.5."""
    import pandas as pd
    df = pd.read_csv(common.aos_file("coris.dat"), skiprows=[1, 2, 3], skipinitialspace=True)
    return df.loc[df.chd == 0, "ldl"].to_numpy(float), df.loc[df.chd == 1, "ldl"].to_numpy(float)


# =========================================================================== plug-in statistics
def skewness(x, axis=-1):
    """Plug-in skewness T(F_n) = n^-1 sum (X_i - mean)^3 / sigma_hat^3 (divisor n), along axis."""
    x = np.asarray(x, float)
    c = x - x.mean(axis=axis, keepdims=True)
    m2 = (c * c).mean(axis=axis)
    return (c ** 3).mean(axis=axis) / m2 ** 1.5


def corr_rows(Y, Zm):
    """Sample correlation of each row pair of Y and Zm (arrays of the same shape)."""
    Y = np.asarray(Y, float)
    Zm = np.asarray(Zm, float)
    a = Y - Y.mean(-1, keepdims=True)
    b = Zm - Zm.mean(-1, keepdims=True)
    return (a * b).sum(-1) / np.sqrt((a * a).sum(-1) * (b * b).sum(-1))


def ratio_of_means(Y, Zm):
    return np.asarray(Y, float).mean(-1) / np.asarray(Zm, float).mean(-1)


def q7(sorted_x, p):
    """Sample p-quantile by linear interpolation between order statistics (R's type 7, numpy's
    default), written out so the browser demos can use the identical formula:
    h = (B - 1) p, j = floor(h), value x_(j) + (h - j)(x_(j+1) - x_(j)) with 0-based order statistics."""
    s = np.asarray(sorted_x, float)
    h = (s.shape[-1] - 1) * p
    j = int(math.floor(h))
    if j >= s.shape[-1] - 1:
        return s[..., -1]
    return s[..., j] + (h - j) * (s[..., j + 1] - s[..., j])


def iqr_scale(x, axis=-1):
    """(q.75 - q.25) / 1.34 of Exercise 8.3 (type 7 quantiles), along the last axis."""
    s = np.sort(np.asarray(x, float), axis=axis)
    return (q7(s, 0.75) - q7(s, 0.25)) / 1.34


def median(x, axis=-1):
    return np.median(np.asarray(x, float), axis=axis)


# =========================================================================== Section 8.2: the bootstrap
def bootstrap(stat, data, B, rng):
    """B bootstrap replications T*_1, ..., T*_B of stat (Steps 1-3 of the book's box).

    data is a 1-D array of n observations, or a tuple of 1-D arrays of the same length n (paired data,
    resampled as pairs). stat must act along the last axis, so one call evaluates all B resamples.
    Drawing n times with replacement from the data is drawing an iid sample from F_n."""
    arrays = data if isinstance(data, tuple) else (data,)
    n = len(arrays[0])
    idx = rng.integers(0, n, size=(B, n))
    return stat(*[np.asarray(a)[idx] for a in arrays])


def v_boot(reps):
    """Equation (8.1): the variance of the bootstrap replications, with divisor B."""
    reps = np.asarray(reps, float)
    return float(np.mean((reps - reps.mean()) ** 2))


def boot_intervals(theta, reps, alpha=0.05):
    """The three 1 - alpha intervals of Section 8.3 from the estimate and its bootstrap replications:
    Normal (8.2), pivotal (8.6) and percentile. Quantiles by linear interpolation (q7)."""
    s = np.sort(np.asarray(reps, float))
    se = math.sqrt(v_boot(s))
    z = Z.get(alpha, float(stats.norm.ppf(1 - alpha / 2)))
    lo, hi = float(q7(s, alpha / 2)), float(q7(s, 1 - alpha / 2))
    return {"se": se, "normal": (theta - z * se, theta + z * se), "pivotal": (2 * theta - hi, 2 * theta - lo),
            "percentile": (lo, hi)}


def boot_median_exact(x):
    """The ideal (B = infinity) bootstrap distribution of the median for odd n = 2m - 1.

    The median of a resample is X*_(m), and X*_(m) <= v exactly when at least m of the n draws are
    <= v; the number of such draws is Binomial(n, F_n(v)). Hence P*(median* <= v) = P(Bin(n, F_n(v)) >= m)
    at each distinct data value v (ties are allowed). Returns (values, probabilities)."""
    x = np.sort(np.asarray(x, float))
    n = x.size
    if n % 2 == 0:
        raise ValueError("boot_median_exact needs odd n (the median is then an order statistic)")
    m = (n + 1) // 2
    v = np.unique(x)
    Fn = np.searchsorted(x, v, side="right") / n
    cdf = stats.binom.sf(m - 1, n, Fn)
    cdf[-1] = 1.0
    return v, np.diff(np.concatenate([[0.0], cdf]))


def dist_moments(v, p):
    mu = float(np.sum(v * p))
    return mu, float(np.sum((v - mu) ** 2 * p))


def dist_quantile(v, p, beta):
    """Generalised inverse inf{v : P(V <= v) >= beta} of a discrete distribution."""
    c = np.cumsum(p)
    return float(v[np.searchsorted(c, beta - 1e-12)])


def mc_rel_sd(reps):
    """Approximate relative Monte Carlo standard deviation of se_boot = sqrt(v_boot) for this B:
    sd(v_boot) ~ sqrt((mu4 - v^2) / B), and the delta method halves the relative error for the root,
    giving sqrt((kappa - 1) / (4 B)) with kappa the kurtosis of the bootstrap distribution."""
    r = np.asarray(reps, float)
    c = r - r.mean()
    kap = np.mean(c ** 4) / np.mean(c ** 2) ** 2
    return math.sqrt((kap - 1) / (4 * r.size)), kap


def se_influence_skew(x):
    """Nonparametric delta method se of the plug-in skewness: sqrt(mean(L_hat^2) / n), with the
    empirical influence function L(x) = [(x-mu)^3 - mu3 - 3 sigma^2 (x-mu)] / sigma^3
    - (3/2) mu3 [(x-mu)^2 - sigma^2] / sigma^5."""
    x = np.asarray(x, float)
    c = x - x.mean()
    s2, m3 = np.mean(c ** 2), np.mean(c ** 3)
    L = (c ** 3 - m3 - 3 * s2 * c) / s2 ** 1.5 - 1.5 * m3 * (c ** 2 - s2) / s2 ** 2.5
    return math.sqrt(np.mean(L ** 2) / x.size), L


# =========================================================================== Appendix 8.5.1: the jackknife
def jackknife(stat, data):
    """v_jack = (n-1)/n sum_i (T_(-i) - T_bar)^2 and the leave-one-out values T_(-i).
    data is a 1-D array or a tuple of paired 1-D arrays; stat acts along the last axis."""
    arrays = data if isinstance(data, tuple) else (data,)
    n = len(arrays[0])
    keep = ~np.eye(n, dtype=bool)
    loo = stat(*[np.broadcast_to(np.asarray(a, float), (n, n))[keep].reshape(n, n - 1) for a in arrays])
    return float((n - 1) / n * np.sum((loo - loo.mean()) ** 2)), loo


def median_var_exp(n):
    """Exact variance of the median X_(m), n = 2m - 1 odd, of n iid Exponential(1) variables:
    by Renyi's representation X_(k) = sum_{j=n-k+1}^{n} E_j / j with E_j iid Exp(1)."""
    m = (n + 1) // 2
    j = np.arange(n - m + 1, n + 1, dtype=float)
    return float(np.sum(1.0 / j ** 2))


# =========================================================================== Exercises
def n_distinct_resamples(n):
    """Exercise 8.4: the number of distinct bootstrap samples (multisets of size n from n distinct
    points) is C(2n - 1, n)."""
    return math.comb(2 * n - 1, n)


def boot_mean_moments(x):
    """Exercise 8.5 (conditional part): E(mean* | X) = mean, V(mean* | X) = sigma_hat^2 / n (divisor n)."""
    x = np.asarray(x, float)
    return float(x.mean()), float(np.mean((x - x.mean()) ** 2) / x.size)


def vboot_mean_square(x):
    """Exercise 8.8: the ideal bootstrap variance of T_n = mean^2, exactly, and the book's formula.

    With a_k = n^-1 sum (X_i - mean)^k (signed central moments of F_n):
      correct = 4 mean^2 a2/n + 4 mean a3/n^2 + a4/n^3 + (2n - 3) a2^2 / n^3,
      book    = 4 mean^2 a2/n + 4 mean a3/n^2 + a4/n^3   (the printed formula, missing the last term)."""
    x = np.asarray(x, float)
    n, m = x.size, x.mean()
    a2, a3, a4 = (np.mean((x - m) ** k) for k in (2, 3, 4))
    book = 4 * m * m * a2 / n + 4 * m * a3 / n ** 2 + a4 / n ** 3
    return float(book + (2 * n - 3) * a2 ** 2 / n ** 3), float(book)


# =========================================================================== coverage experiments
def coverage_study(draw, stat, theta, n, R, B, rng, alpha=0.05):
    """R simulated data sets of size n from draw(rng, n); for each, B bootstrap replications and the
    three intervals. Returns per-method arrays: covered, missed low (theta < lower end), missed high,
    and lengths."""
    out = {k: {"cover": np.zeros(R, bool), "low": np.zeros(R, bool), "high": np.zeros(R, bool),
               "len": np.zeros(R)} for k in ("normal", "pivotal", "percentile")}
    for r in range(R):
        x = draw(rng, n)
        t = float(stat(x))
        reps = bootstrap(stat, x, B, rng)
        ci = boot_intervals(t, reps, alpha)
        for k in out:
            a, b = ci[k]
            out[k]["cover"][r] = a <= theta <= b
            out[k]["low"][r] = theta < a
            out[k]["high"][r] = theta > b
            out[k]["len"][r] = b - a
    return out


SKEW_LOGNORMAL = (math.e + 2) * math.sqrt(math.e - 1)          # skewness of exp(N(0, 1))
IQR_T3 = 2 * float(stats.t.ppf(0.75, 3)) / 1.34                # (q.75 - q.25)/1.34 for t_3


def _cached(name, fn):
    os.makedirs(CACHE_DIR, exist_ok=True)
    path = os.path.join(CACHE_DIR, name + ".npz")
    if os.path.exists(path):
        return dict(np.load(path, allow_pickle=True))
    res = fn()
    np.savez(path, **res)
    return res


# =========================================================================== the demos' generator
def js_uniforms(seed, n, start=0):
    """Uniforms number start+1, ..., start+n of DK.rng(seed) (courses/demo-kit.js), vectorised:
    mulberry32 adds a constant to its state at each call, so the j-th output is a hash of
    seed + j * 0x6D2B79F5 (mod 2^32)."""
    j = np.arange(start + 1, start + n + 1, dtype=np.uint64)
    m = np.uint64(M32)
    a = (np.uint64(int(seed) & M32) + j * np.uint64(0x6D2B79F5)) & m
    t = ((a ^ (a >> np.uint64(15))) * (np.uint64(1) | a)) & m
    t = ((t + (((t ^ (t >> np.uint64(7))) * (np.uint64(61) | t)) & m)) & m) ^ t
    return ((t ^ (t >> np.uint64(14))) & m).astype(np.float64) / 4294967296.0


def js_normals(seed, n, start=0):
    """n consecutive DK.rng(seed).norm() values starting after `start` uniforms (Box-Muller pairs:
    r cos first, then the cached r sin); they use 2 * ceil(n / 2) uniforms."""
    npair = (n + 1) // 2
    u = js_uniforms(seed, 2 * npair, start)
    u1, u2 = u[0::2], u[1::2]
    if np.any(u1 == 0.0):
        raise RuntimeError("a zero uniform in the stream")
    r = np.sqrt(-2.0 * np.log(u1))
    th = 2.0 * np.pi * u2
    out = np.empty(2 * npair)
    out[0::2] = r * np.cos(th)
    out[1::2] = r * np.sin(th)
    return out[:n]


def js_data(pop, seed, n, pool=None):
    """A data set as the demos draw it from DK.rng(seed); returns (x, number of uniforms used).
    pop: 'lnorm' exp(Z); 'exp' -log(U); 'norm' Z; 't3' Z / sqrt((Z1^2 + Z2^2 + Z3^2) / 3);
    'unif' U; 'nerve' n draws with replacement from pool (the nerve data)."""
    if pop in ("lnorm", "norm"):
        z = js_normals(seed, n)
        return (np.exp(z) if pop == "lnorm" else z), 2 * ((n + 1) // 2)
    if pop == "t3":
        z = js_normals(seed, 4 * n).reshape(n, 4)
        return z[:, 0] / np.sqrt((z[:, 1] ** 2 + z[:, 2] ** 2 + z[:, 3] ** 2) / 3.0), 4 * n
    u = js_uniforms(seed, n)
    if pop == "exp":
        if np.any(u == 0.0):
            raise RuntimeError("a zero uniform in the stream")
        return -np.log(u), n
    if pop == "unif":
        return u, n
    if pop == "nerve":
        return np.asarray(pool)[np.floor(u * len(pool)).astype(int)], n
    raise ValueError(pop)


STATS = {"mean": lambda x: np.asarray(x).mean(-1), "median": median, "skew": skewness, "iqr": iqr_scale,
         "max": lambda x: np.asarray(x).max(-1)}

# The problems of the coverage demo: (population, statistic, true value).
COV_PROBLEMS = {
    "lnskew": ("lnorm", "skew", SKEW_LOGNORMAL),
    "expskew": ("exp", "skew", 2.0),
    "lnmean": ("lnorm", "mean", math.exp(0.5)),
    "expmean": ("exp", "mean", 1.0),
    "t3iqr": ("t3", "iqr", IQR_T3),
    "expmed": ("exp", "median", math.log(2.0)),
    "umax": ("unif", "max", 1.0),
}


def js_boot(x, stat, seed, B, start):
    """B bootstrap replications as the demos draw them: resample b uses uniforms start + b n + 1, ...,
    start + (b + 1) n of DK.rng(seed), index floor(u n)."""
    n = x.size
    idx = np.floor(js_uniforms(seed, B * n, start) * n).astype(int).reshape(B, n)
    return STATS[stat](x[idx])


def js_intervals(theta, reps, alpha):
    """boot_intervals with the variance computed as in the demos (two passes, divisor B)."""
    return boot_intervals(theta, reps, alpha)


def cov_demo(problem, n, B, R, alpha, seed):
    """The coverage demo: data set r comes from DK.rng(seed + 7919 r), and its B resamples continue
    the same stream. Returns counts of covered / missed-low / missed-high and mean lengths per method,
    plus the first data set's estimate, se and intervals."""
    pop, st, theta = COV_PROBLEMS[problem]
    res = {k: {"cover": 0, "low": 0, "high": 0, "len": 0.0} for k in ("normal", "pivotal", "percentile")}
    first = None
    for r in range(R):
        s = seed + 7919 * r
        x, used = js_data(pop, s, n)
        t = float(STATS[st](x))
        reps = js_boot(x, st, s, B, used)
        ci = js_intervals(t, reps, alpha)
        if r == 0:
            first = {"theta_hat": t, "se": ci["se"], "normal": ci["normal"], "pivotal": ci["pivotal"],
                     "percentile": ci["percentile"]}
        for k in res:
            a, b = ci[k]
            res[k]["cover"] += int(a <= theta <= b)
            res[k]["low"] += int(theta < a)
            res[k]["high"] += int(theta > b)
            res[k]["len"] += (b - a) / R
    return res, first


def jk_demo(pop, stat, n, B, R, seed, M=4000, pool=None):
    """The jackknife-versus-bootstrap demo. Data set r from DK.rng(seed + 7919 r); its B resamples
    continue the stream. The true se is the sd (divisor M) of the statistic over M data sets drawn
    from DK.rng(500000 + 7919 j), j = 0..M-1 (fixed, so it does not change with seed).
    Returns se_true and arrays se_jack, se_boot over the R data sets."""
    f = STATS[stat]
    tv = np.array([f(js_data(pop, 500000 + 7919 * j, n, pool)[0]) for j in range(M)])
    se_true = math.sqrt(np.mean((tv - tv.mean()) ** 2))
    sj, sb = np.zeros(R), np.zeros(R)
    for r in range(R):
        s = seed + 7919 * r
        x, used = js_data(pop, s, n, pool)
        sj[r] = math.sqrt(jackknife(f, x)[0])
        sb[r] = math.sqrt(v_boot(js_boot(x, stat, s, B, used)))
    return se_true, sj, sb


def breps_grid(bmax):
    """The values of B at which the replicates demo records its running estimates: 10, 11, ..., 20
    and then about 30 sizes per decade, always including bmax."""
    g = set(range(10, min(bmax, 20) + 1))
    v = 20.0
    while v < bmax:
        v *= 1.0797751623277096          # 10^(1/30), written out so the browser uses the same number
        g.add(min(bmax, int(math.floor(v + 0.5))))   # Math.round in JavaScript
    g.add(bmax)
    return sorted(g)


def breps_demo(problem, K, bmax, seed, data):
    """The replicates demo: run k uses DK.rng(seed + 7919 k) and draws resample b from uniforms
    b n + 1, ..., (b + 1) n. Returns the grid and, per run, the running se_boot (divisor B, centred at
    the full-data estimate for numerical stability, which leaves the variance unchanged) and the
    running 2.5% and 97.5% quantiles of the replications."""
    x, y, st = data[problem]
    grid = breps_grid(bmax)
    out = {"grid": grid, "se": [], "lo": [], "hi": []}
    pooled = []
    for k in range(K):
        s = seed + 7919 * k
        n = x.size
        idx = np.floor(js_uniforms(s, bmax * n) * n).astype(int).reshape(bmax, n)
        reps = st(x[idx]) if y is None else st(x[idx], y[idx])
        t0 = float(st(x) if y is None else st(x, y))
        d = reps - t0
        c1, c2 = np.cumsum(d), np.cumsum(d * d)
        se, lo, hi = [], [], []
        for b in grid:
            m = c1[b - 1] / b
            se.append(math.sqrt(max(c2[b - 1] / b - m * m, 0.0)))
            srt = np.sort(reps[:b])
            lo.append(float(q7(srt, 0.025)))
            hi.append(float(q7(srt, 0.975)))
        out["se"].append(se)
        out["lo"].append(lo)
        out["hi"].append(hi)
        pooled.append(reps)
    c = np.concatenate(pooled)
    c = c - c.mean()
    out["kappa"] = float(np.mean(c ** 4) / np.mean(c ** 2) ** 2)    # kurtosis of the pooled replications
    return out


def breps_data():
    """The five problems of the replicates demo: (x, y or None, statistic)."""
    xn = nerve()
    yp, zp = patch_yz()
    return {"nmed": (xn, None, median), "nmean": (xn, None, lambda a: np.asarray(a).mean(-1)),
            "nskew": (xn, None, skewness), "law": (LSAT, GPA, corr_rows), "patch": (yp, zp, ratio_of_means)}


def breps_reference(Bref=200000, seed=8008):
    """Reference (near-ideal) bootstrap se and 2.5%, 97.5% quantiles for the replicates demo: exact
    for the nerve median (boot_median_exact) and the se of the nerve mean, a large-B simulation otherwise."""
    rng = np.random.default_rng(seed)
    ref = {}
    for key, (x, y, st) in breps_data().items():
        if key == "nmed":
            v, p = boot_median_exact(x)
            ref[key] = {"se": math.sqrt(dist_moments(v, p)[1]), "lo": dist_quantile(v, p, 0.025),
                        "hi": dist_quantile(v, p, 0.975), "exact": True}
            continue
        reps = []
        for _ in range(Bref // 5000):
            reps.append(bootstrap(st, x if y is None else (x, y), 5000, rng))
        reps = np.sort(np.concatenate(reps))
        se = math.sqrt(v_boot(reps)) if key != "nmean" else math.sqrt(np.mean((x - x.mean()) ** 2) / x.size)
        ref[key] = {"se": se, "lo": float(q7(reps, 0.025)), "hi": float(q7(reps, 0.975)), "exact": key == "nmean",
                    "B": Bref}
    return ref


def export_demo_data():
    """../data/ch08-bootstrap.json: the nerve, law school and patch data and the reference values of
    the replicates demo."""
    ref = _cached_json("breps_reference", breps_reference)
    yp, zp = patch_yz()
    out = {"note": "Data of Wasserman (2004), Chapter 8: nerve.dat from the book's website (799 waiting times "
                   "in seconds between nerve pulses), the law school data of Example 8.6 (GPA of the last school "
                   "2.96, as corrected in the author's errata) and the patch data of Example 8.7 "
                   "(y = new - old, z = old - placebo). ref: near-ideal bootstrap se and 2.5%/97.5% quantiles "
                   "(exact for the median and for the se of the mean; otherwise from 200000 replications).",
           "nerve": [float(v) for v in nerve()], "lsat": LSAT.tolist(), "gpa": GPA.tolist(),
           "patch_y": yp.tolist(), "patch_z": zp.tolist(), "ref": ref}
    os.makedirs(os.path.dirname(DATA_OUT), exist_ok=True)
    with open(DATA_OUT, "w") as f:
        json.dump(out, f, separators=(",", ":"))
    print("wrote", os.path.relpath(DATA_OUT, os.getcwd()))
    return out


def _cached_json(name, fn):
    os.makedirs(CACHE_DIR, exist_ok=True)
    path = os.path.join(CACHE_DIR, name + ".json")
    if os.path.exists(path):
        with open(path) as f:
            return json.load(f)
    res = fn()
    with open(path, "w") as f:
        json.dump(res, f)
    return res


# =========================================================================== figures
def _hist(ax, v, bins, color, label=None, density=False, alpha=0.75):
    ax.hist(v, bins=bins, color=color, alpha=alpha, density=density, label=label, edgecolor="white", linewidth=0.4)


def fig_8_1(out_dir=None):
    """Figure 8.1: the law school data and B = 1000 bootstrap correlations."""
    rng = np.random.default_rng(81)
    theta = float(corr_rows(LSAT, GPA))
    reps = bootstrap(corr_rows, (LSAT, GPA), 1000, rng)
    ci = boot_intervals(theta, reps)
    fig, axes = common.panel_grid(1, 2, size=(3.9, 3.0))
    ax = axes[0, 0]
    ax.plot(LSAT, GPA, "o", color=BLUE, ms=5)
    ax.set_xlabel("LSAT")
    ax.set_ylabel("GPA")
    ax.set_title(r"Law school data, $\hat\theta=%.3f$" % theta)
    common.panel_label(ax, "a", loc="lower right")
    ax = axes[0, 1]
    _hist(ax, reps, np.linspace(0.1, 1.0, 37), BLUE)
    ax.axvline(theta, color="k", lw=1)
    for k, c, y in (("normal", GREEN, 0.93), ("percentile", RED, 0.85), ("pivotal", ORANGE, 0.77)):
        a, b = ci[k]
        ax.plot([a, min(b, 1.08)], [y * ax.get_ylim()[1]] * 2, color=c, lw=2.2, label=k)
    ax.axvline(1.0, color=GREY, lw=0.8, ls=":")
    ax.set_xlim(0.1, 1.12)
    ax.set_xlabel(r"bootstrap correlation $\hat\theta^*$")
    ax.set_ylabel("count")
    ax.set_title(r"$B=1000$ replications, $\widehat{\mathsf{se}}=%.3f$" % ci["se"])
    ax.legend(fontsize=7.5, frameon=False, loc="upper left")
    common.panel_label(ax, "b", loc="lower left")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-1", out_dir)


def fig_8_2(out_dir=None):
    """Figure 8.2: B = 1000 bootstrap replications of the patch ratio."""
    rng = np.random.default_rng(82)
    y, z = patch_yz()
    theta = float(ratio_of_means(y, z))
    reps = bootstrap(ratio_of_means, (y, z), 1000, rng)
    ci = boot_intervals(theta, reps)
    fig, axes = common.panel_grid(1, 1, size=(4.6, 3.0))
    ax = axes[0, 0]
    _hist(ax, reps, np.linspace(-0.35, 0.25, 31), BLUE)
    ax.axvline(theta, color="k", lw=1)
    for v in (-0.2, 0.2):
        ax.axvline(v, color=MAGENTA, lw=1, ls="--")
    top = ax.get_ylim()[1]
    for k, c, yy in (("normal", GREEN, 0.95), ("percentile", RED, 0.88), ("pivotal", ORANGE, 0.81)):
        a, b = ci[k]
        ax.plot([a, b], [yy * top] * 2, color=c, lw=2.2, label="%s (%.2f, %.2f)" % (k, a, b))
    ax.set_xlabel(r"bootstrap ratio $\hat\theta^*=\bar Y^*/\bar Z^*$")
    ax.set_ylabel("count")
    ax.set_title(r"Patch data: $\hat\theta=%.4f$, $\widehat{\mathsf{se}}=%.3f$" % (theta, ci["se"]))
    ax.legend(fontsize=7, frameon=False, loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-2", out_dir)


def fig_8_s1(out_dir=None):
    """Figure 8.S1: the two approximations, Monte Carlo error (B) and statistical error (n)."""
    rng = np.random.default_rng(801)
    x = nerve()
    v, p = boot_median_exact(x)
    se_ideal = math.sqrt(dist_moments(v, p)[1])
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.75))
    ax = axes[0, 0]
    Bs = np.unique(np.round(np.logspace(1, np.log10(5000), 60)).astype(int))
    for k in range(12):
        reps = bootstrap(median, x, 5000, rng)
        d = reps - np.median(x)
        c1, c2 = np.cumsum(d), np.cumsum(d * d)
        se = np.sqrt(np.maximum(c2[Bs - 1] / Bs - (c1[Bs - 1] / Bs) ** 2, 0))
        ax.plot(Bs, se, color=BLUE, lw=0.8, alpha=0.6)
    # the Monte Carlo band +-2 sd from the kurtosis of the exact bootstrap distribution
    mu = np.sum(v * p)
    kap = np.sum((v - mu) ** 4 * p) / np.sum((v - mu) ** 2 * p) ** 2
    band = se_ideal * 2 * np.sqrt((kap - 1) / (4 * Bs))
    ax.plot(Bs, se_ideal + band, color=ORANGE, ls="--", lw=1)
    ax.plot(Bs, se_ideal - band, color=ORANGE, ls="--", lw=1)
    ax.axhline(se_ideal, color="k", lw=1.2)
    ax.set_xscale("log")
    ax.set_xlabel("number of replications B")
    ax.set_ylabel(r"$\widehat{\mathsf{se}}_{\rm boot}$ of the median")
    ax.set_title("Nerve data: Monte Carlo error")
    common.panel_label(ax, "a", loc="upper right")
    # (b) statistical error: the ideal bootstrap se of the median over data sets, Exp(1) data
    ax = axes[0, 1]
    for n, c in ((25, RED), (101, BLUE), (401, GREEN)):
        true = math.sqrt(median_var_exp(n))
        r = []
        for _ in range(1000):
            vv, pp = boot_median_exact(rng.exponential(size=n))
            r.append(math.sqrt(dist_moments(vv, pp)[1]) / true)
        _hist(ax, r, np.linspace(0, 2.6, 53), c, label="n = %d" % n, density=True, alpha=0.45)
    ax.axvline(1, color="k", lw=1)
    ax.set_xlabel(r"ideal $\widehat{\mathsf{se}}_{\rm boot}$ / true se (median)")
    ax.set_ylabel("density")
    ax.set_title("Exponential data: statistical error")
    ax.legend(fontsize=7.5, frameon=False)
    common.panel_label(ax, "b", loc="upper left")
    # (c) the ideal bootstrap distribution of the nerve median (exact) against B = 1000 replications
    ax = axes[0, 2]
    reps = bootstrap(median, x, 1000, rng)
    vals, cnt = np.unique(reps, return_counts=True)
    ax.vlines(v, 0, p, color=BLUE, lw=3, label="exact (B = infinity)")
    ax.plot(vals, cnt / 1000, "o", color=RED, ms=3.5, label="B = 1000")
    ax.set_xlim(0.11, 0.19)
    ax.set_xlabel(r"bootstrap median $T^*_n$ (seconds)")
    ax.set_ylabel("probability")
    ax.set_title("Exact bootstrap distribution")
    ax.legend(fontsize=7.5, frameon=False, loc="center right")
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-S1", out_dir)


def fig_8_s2(out_dir=None):
    """Figure 8.S2: bootstrap for the skewness of the nerve data (Examples 8.2 and 8.4)."""
    rng = np.random.default_rng(802)
    x = nerve()
    t = float(skewness(x))
    reps = bootstrap(skewness, x, 1000, rng)
    ci = boot_intervals(t, reps)
    fig, axes = common.panel_grid(1, 2, size=(4.1, 2.9))
    ax = axes[0, 0]
    bins = np.linspace(1.2, 2.4, 49)
    _hist(ax, reps, bins, BLUE, density=True)
    ax.axvline(t, color="k", lw=1)
    ax.set_ylim(0, 1.45 * ax.get_ylim()[1])
    top = ax.get_ylim()[1]
    for k, c, yy in (("normal", GREEN, 0.93), ("pivotal", ORANGE, 0.85), ("percentile", RED, 0.77)):
        a, b = ci[k]
        ax.plot([a, b], [yy * top] * 2, color=c, lw=2.4)
        ax.text(b + 0.02, yy * top, "%s (%.2f, %.2f)" % (k, a, b), va="center", fontsize=7, color=c)
    g = np.linspace(1.2, 2.4, 300)
    ax.plot(g, stats.norm.pdf(g, t, ci["se"]), color=GREEN, lw=1, ls="--")
    ax.set_xlim(1.2, 2.75)
    ax.set_xlabel(r"bootstrap skewness $\hat\theta^*$")
    ax.set_ylabel("density")
    ax.set_title(r"Nerve data: $\hat\theta=%.3f$, $\widehat{\mathsf{se}}_{\rm boot}=%.3f$" % (t, ci["se"]))
    common.panel_label(ax, "a", loc="upper left")
    # (b) the law school correlation: a skewed bootstrap distribution and its reflection
    ax = axes[0, 1]
    tl = float(corr_rows(LSAT, GPA))
    rl = bootstrap(corr_rows, (LSAT, GPA), 4000, rng)
    cl = boot_intervals(tl, rl)
    bins = np.linspace(0.1, 1.45, 55)
    _hist(ax, rl, bins, BLUE, density=True, alpha=0.5, label=r"$\hat\theta^*$: percentile interval")
    _hist(ax, 2 * tl - rl, bins, ORANGE, density=True, alpha=0.5, label=r"$2\hat\theta-\hat\theta^*$: pivotal interval")
    ax.axvline(tl, color="k", lw=1)
    ax.axvline(1.0, color=GREY, lw=1, ls=":")
    ax.set_ylim(0, 1.5 * ax.get_ylim()[1])
    top = ax.get_ylim()[1]
    for k, c, yy in (("percentile", BLUE, 0.70), ("pivotal", ORANGE, 0.63)):
        a, b = cl[k]
        ax.plot([a, b], [yy * top] * 2, color=c, lw=2.4)
        ax.text(b + 0.02, yy * top, "(%.2f, %.2f)" % (a, b), va="center", fontsize=7, color=c)
    ax.set_xlabel("law school correlation")
    ax.set_title("The pivotal interval reflects")
    ax.legend(fontsize=7, frameon=False, loc="upper left")
    common.panel_label(ax, "b", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-S2", out_dir)


def coverage_ex2(R=1000, B=1000, ns=(25, 50, 100, 200), seed=8200):
    """Exercise 8.2: coverage of the three intervals for the skewness of exp(N(0,1)) data."""
    def run():
        out = {}
        for n in ns:
            rng = np.random.default_rng(seed + n)
            res = coverage_study(lambda g, m: np.exp(g.standard_normal(m)), skewness, SKEW_LOGNORMAL, n, R, B, rng)
            for k, d in res.items():
                for q, arr in d.items():
                    out["%d_%s_%s" % (n, k, q)] = arr
        return out
    return _cached("cov_ex2_R%d_B%d" % (R, B), run)


def coverage_ex3(R=1000, B=1000, n=25, seed=8300):
    """Exercise 8.3: coverage and length for (q.75 - q.25)/1.34 with t_3 data, n = 25."""
    def run():
        rng = np.random.default_rng(seed)
        res = coverage_study(lambda g, m: g.standard_t(3, m), iqr_scale, IQR_T3, n, R, B, rng)
        return {"%s_%s" % (k, q): arr for k, d in res.items() for q, arr in d.items()}
    return _cached("cov_ex3_R%d_B%d" % (R, B), run)


METHODS = (("normal", GREEN), ("pivotal", ORANGE), ("percentile", RED))


def fig_8_s3(out_dir=None):
    """Figure 8.S3: coverage of the three intervals for a skewed statistic (Exercise 8.2)."""
    ns = (25, 50, 100, 200)
    cv = coverage_ex2()
    R = len(cv["25_normal_cover"])
    fig, axes = common.panel_grid(1, 2, size=(4.0, 2.9))
    ax = axes[0, 0]
    for k, c in METHODS:
        cov = [np.mean(cv["%d_%s_cover" % (n, k)]) for n in ns]
        ax.plot(ns, cov, "o-", color=c, label=k)
    ax.axhline(0.95, color="k", lw=0.9, ls="--")
    ax.set_xscale("log")
    ax.set_xticks(ns)
    ax.set_xticklabels([str(n) for n in ns])
    ax.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter())
    ax.set_ylim(0, 1)
    ax.set_xlabel("sample size n")
    ax.set_ylabel("coverage")
    ax.set_title("Skewness of lognormal data")
    ax.legend(fontsize=7.5, frameon=False, loc="center right")
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    n = 50
    xs = np.arange(3)
    low = [np.mean(cv["%d_%s_low" % (n, k)]) for k, _ in METHODS]
    high = [np.mean(cv["%d_%s_high" % (n, k)]) for k, _ in METHODS]
    cov = [np.mean(cv["%d_%s_cover" % (n, k)]) for k, _ in METHODS]
    ax.bar(xs, cov, color=[c for _, c in METHODS], alpha=0.85, label="covers")
    ax.bar(xs, high, bottom=cov, color=GREY, alpha=0.55, label=r"too low ($\theta$ above)")
    ax.bar(xs, low, bottom=np.add(cov, high), color=PURPLE, alpha=0.7, label=r"too high ($\theta$ below)")
    for i in range(3):
        ax.text(xs[i], cov[i] / 2, "%.3f" % cov[i], ha="center", va="center", fontsize=8, color="white")
    ax.set_xticks(xs)
    ax.set_xticklabels([k for k, _ in METHODS])
    ax.set_ylim(0, 1)
    ax.set_ylabel("fraction of %d data sets" % R)
    ax.set_title("n = 50: where the intervals miss")
    ax.legend(fontsize=7, frameon=False, loc="upper center", bbox_to_anchor=(0.5, -0.13), ncol=2)
    common.panel_label(ax, "b", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-S3", out_dir)


def fig_8_s4(out_dir=None):
    """Figure 8.S4: coverage and length for the scaled IQR of t_3 data (Exercise 8.3)."""
    cv = coverage_ex3()
    R = len(cv["normal_cover"])
    fig, axes = common.panel_grid(1, 2, size=(3.6, 2.8))
    ax = axes[0, 0]
    xs = np.arange(3)
    cov = [np.mean(cv["%s_cover" % k]) for k, _ in METHODS]
    err = [2 * math.sqrt(c * (1 - c) / R) for c in cov]
    ax.bar(xs, cov, yerr=err, color=[c for _, c in METHODS], alpha=0.85, capsize=4)
    for i in range(3):
        ax.text(xs[i], 0.05, "%.3f" % cov[i], ha="center", fontsize=8, color="white")
    ax.axhline(0.95, color="k", lw=0.9, ls="--")
    ax.set_xticks(xs)
    ax.set_xticklabels([k for k, _ in METHODS])
    ax.set_ylim(0, 1.02)
    ax.set_ylabel("coverage")
    ax.set_title(r"$t_3$, n = 25: coverage")
    common.panel_label(ax, "a", loc="upper right")
    ax = axes[0, 1]
    data = [cv["%s_len" % k] for k, _ in METHODS]
    bp = ax.boxplot(data, widths=0.55, patch_artist=True, showfliers=False)
    for patch, (_, c) in zip(bp["boxes"], METHODS):
        patch.set_facecolor(c)
        patch.set_alpha(0.6)
    for i, d in enumerate(data):
        ax.plot(i + 1, np.mean(d), "o", color="k", ms=4)
    ax.set_xticks([1, 2, 3])
    ax.set_xticklabels([k for k, _ in METHODS])
    ax.set_ylabel("interval length")
    ax.set_title("length (dot: mean)")
    common.panel_label(ax, "b", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-S4", out_dir)


def jack_vs_boot(stat, draw, n, R, B, se_true, rng):
    sj, sb = np.zeros(R), np.zeros(R)
    for r in range(R):
        x = draw(rng, n)
        sj[r] = math.sqrt(jackknife(stat, x)[0])
        sb[r] = math.sqrt(v_boot(bootstrap(stat, x, B, rng)))
    return sj / se_true, sb / se_true


def jack_median_study(ns=(21, 51, 101, 201, 401), R=1000, seed=8500):
    """Ratios se_jack / se_true and ideal se_boot / se_true for the median of Exp(1) data."""
    def run():
        out = {}
        for n in ns:
            rng = np.random.default_rng(seed + n)
            true = math.sqrt(median_var_exp(n))
            rj, rb = np.zeros(R), np.zeros(R)
            for r in range(R):
                x = rng.exponential(size=n)
                rj[r] = math.sqrt(jackknife(median, x)[0]) / true
                vv, pp = boot_median_exact(x)
                rb[r] = math.sqrt(dist_moments(vv, pp)[1]) / true
            out["jack_%d" % n], out["boot_%d" % n] = rj, rb
        return out
    return _cached("jack_median_R%d" % R, run)


def fig_8_s5(out_dir=None):
    """Figure 8.S5: the jackknife against the bootstrap (Appendix 8.5.1)."""
    ns = (21, 51, 101, 201, 401)
    st = jack_median_study(ns)
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.75))
    ax = axes[0, 0]
    bins = np.linspace(0, 4, 61)
    _hist(ax, st["jack_401"], bins, RED, density=True, alpha=0.5, label="jackknife")
    _hist(ax, st["boot_401"], bins, BLUE, density=True, alpha=0.5, label="bootstrap (ideal)")
    g = np.linspace(0.001, 4, 300)
    ax.plot(g, 4 * g * np.exp(-2 * g), color="k", lw=1.2, label="limit: Gamma(2)/2")
    ax.axvline(1, color="k", lw=0.8, ls=":")
    ax.set_xlabel("estimated se / true se")
    ax.set_ylabel("density")
    ax.set_title("Median, Exp(1) data, n = 401")
    ax.legend(fontsize=7, frameon=False)
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    for key, c, lab in (("jack", RED, "jackknife"), ("boot", BLUE, "bootstrap (ideal)")):
        mv = [np.mean(st["%s_%d" % (key, n)] ** 2) for n in ns]
        sd = [np.std(st["%s_%d" % (key, n)]) for n in ns]
        ax.plot(ns, mv, "o-", color=c, label=lab + ": mean of v/v_true")
        ax.plot(ns, sd, "s--", color=c, ms=3.5, label=lab + ": sd of se ratio")
    ax.axhline(1.5, color=RED, lw=0.8, ls=":")
    ax.axhline(1, color="k", lw=0.8, ls=":")
    ax.set_xscale("log")
    ax.set_xticks(ns)
    ax.set_xticklabels([str(n) for n in ns])
    ax.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter())
    ax.set_ylim(0, 2.6)
    ax.set_xlabel("sample size n")
    ax.set_title("Median: as n grows (n odd)")
    ax.legend(fontsize=6.3, frameon=False, loc="upper right")
    common.panel_label(ax, "b", loc="upper left")
    # (c) a smooth statistic: the skewness of the nerve data, subsamples of n = 100 as data sets
    ax = axes[0, 2]
    rng = np.random.default_rng(8501)
    pool = nerve()
    n = 100
    tv = np.array([skewness(pool[rng.integers(0, pool.size, n)]) for _ in range(20000)])
    true = tv.std()
    rj, rb = jack_vs_boot(skewness, lambda g, m: pool[g.integers(0, pool.size, m)], n, 400, 400, true, rng)
    ax.plot(rb, rj, "o", color=PURPLE, ms=2.5, alpha=0.6)
    ax.plot([0, 2.5], [0, 2.5], color="k", lw=0.8)
    ax.axvline(1, color=GREY, lw=0.8, ls=":")
    ax.axhline(1, color=GREY, lw=0.8, ls=":")
    ax.set_xlim(0, 2.5)
    ax.set_ylim(0, 2.5)
    ax.set_xlabel("bootstrap se / true se (B = 400)")
    ax.set_ylabel("jackknife se / true se")
    ax.set_title("Skewness, nerve population, n = 100")
    ax.text(2.4, 0.15, "correlation %.2f" % np.corrcoef(rb, rj)[0, 1], ha="right", fontsize=7.5)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-S5", out_dir)


def fig_8_s6(out_dir=None):
    """Figure 8.S6: Exercises 8.6 (exp of a Normal mean) and 8.7 (the maximum of uniforms)."""
    rng = np.random.default_rng(806)
    fig, axes = common.panel_grid(1, 2, size=(4.0, 2.9))
    ax = axes[0, 0]
    n, mu = 100, 5.0
    x = rng.normal(mu, 1.0, n)
    t = math.exp(x.mean())
    reps = np.exp(bootstrap(lambda a: a.mean(-1), x, 2000, rng))
    bins = np.linspace(100, 210, 56)
    _hist(ax, reps, bins, BLUE, density=True, label="bootstrap (B = 2000)")
    g = np.linspace(100, 210, 400)
    ax.plot(g, stats.lognorm.pdf(g, s=1 / math.sqrt(n), scale=math.exp(mu)), color=RED, lw=1.5, label=r"true law of $e^{\bar X}$")
    ax.axvline(math.exp(mu), color=RED, lw=0.8, ls=":")
    ax.axvline(t, color="k", lw=1)
    ax.set_xlabel(r"$\hat\theta=e^{\bar X}$")
    ax.set_ylabel("density")
    ax.set_title(r"Exercise 8.6: $n=100$, $\mu=5$")
    ax.legend(fontsize=7, frameon=False, loc="upper right")
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    n = 50
    x = rng.uniform(0, 1, n)
    mx = x.max()
    reps = bootstrap(lambda a: a.max(-1), x, 2000, rng)
    preps = mx * rng.uniform(0, 1, (2000, n)).max(1)
    g = np.linspace(0, 5, 400)
    ax.plot(g, 1 - (1 - g / n) ** n, color=RED, lw=2, label=r"true: $n(\theta-\hat\theta)$")
    ax.plot(g, 1 - np.exp(-g), color="k", lw=0.8, ls=":", label="limit Exp(1)")
    for r, c, lab in ((n * (mx - preps) / mx, GREEN, "parametric bootstrap"),
                      (n * (mx - reps) / mx, BLUE, "nonparametric bootstrap")):
        srt = np.sort(r)
        ax.step(np.concatenate([[0], srt]), np.arange(srt.size + 1) / srt.size, where="post", color=c, lw=1.4,
                label=lab + r": $n(\hat\theta-\hat\theta^*)/\hat\theta$")
    frac = np.mean(reps == mx)
    ax.annotate("jump of %.3f at 0:\nreplications equal\nto the observed max" % frac, xy=(0.02, frac * 0.97),
                xytext=(1.3, 0.38), fontsize=7.5, arrowprops=dict(arrowstyle="->", lw=0.7))
    ax.set_xlim(0, 5)
    ax.set_ylim(0, 1.02)
    ax.set_xlabel("scaled error")
    ax.set_ylabel("cdf")
    ax.set_title(r"Exercise 8.7: maximum, $n=50$, $\theta=1$")
    ax.legend(fontsize=6.3, frameon=False, loc="lower right")
    common.panel_label(ax, "b", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig8-S6", out_dir)


def fig_data(out_dir=None):
    """The JSON data file of the browser demonstrations (no figure)."""
    export_demo_data()


FIGURES = {"8.1": fig_8_1, "8.2": fig_8_2, "8.S1": fig_8_s1, "8.S2": fig_8_s2, "8.S3": fig_8_s3,
           "8.S4": fig_8_s4, "8.S5": fig_8_s5, "8.S6": fig_8_s6, "data": fig_data}

if __name__ == "__main__":
    common.run(FIGURES, CH)
