"""Wasserman (2006), Chapter 2: Estimating the cdf and Statistical Functionals.

Companion code for Chapter 2 of L. Wasserman, All of Nonparametric Statistics (Springer, 2006),
pp. 13-25, written from scratch with numpy, scipy and matplotlib, to be read next to the lecture
notes (../ch02-estimating-cdf-functionals.html).

* Section 2.1, the cdf: ecdf, dkw_eps and dkw_band (Theorem 2.6), vc_eps (the half-width that the
  Vapnik-Chervonenkis bound (2.44) gives for the same band, Example 2.45), sup_dist_uniform and
  band_coverage (Exercise 2.3, with the seeded generator of the browser demonstrations).
* Section 2.2-2.3, functionals and influence functions: functional (the plug-in T of a weighted
  empirical distribution, so that T((1 - eps) F_n + eps delta_x) can be evaluated exactly),
  influence (the closed-form empirical influence functions of the mean, variance, skewness,
  correlation, quantiles and F(b) - F(a)), gateaux (the difference quotient of (2.21)),
  delta_se and delta_ci (the nonparametric delta method (2.31)), kde_at (a Gaussian kernel
  density estimate, needed by (2.36) for quantiles) and quantile_if_population.
* Section 2.4, VC theory: picked_sets (every subset of a finite point set that half-lines,
  intervals, half-planes, axis-parallel rectangles or discs pick out, by an exact enumeration),
  shatter_count, sauer and vc_bound.
* JSRandom and js_uniforms reproduce DK.rng of courses/demo-kit.js, so that the simulations and
  random point sets of the demonstrations can be repeated here number for number.
* exercise_numbers: Exercises 2.3, 2.7, 2.8, 2.11, 2.12 and 2.13 on the book's data.

Figures: 2.1 (nerve data, empirical cdf and 95 percent DKW band) is reproduced. The chapter has no
other figure. Four supplementary figures (not in the book) illustrate the notes:
  2.S1 influence functions of six functionals at N(0, 1), closed form against difference quotients;
  2.S2 Exercise 2.11: empirical cdf of the Fiji earthquake magnitudes with its 95 percent band;
  2.S3 the nonparametric delta method at work: (T(F_n) - T(F)) / se_hat against N(0, 1);
  2.S4 VC theory: shatter coefficients against Sauer's bound, and DKW against the VC band.

usage: python3 ch02_estimating_cdf_functionals.py [--fig 2.1 2.S1 ...] [--list]
"""
import itertools
import json
import math
import os
import sys

import numpy as np
from scipy import stats

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import common  # noqa: E402
from common import BLUE, RED, GREEN, ORANGE, PURPLE, GREY, savefig  # noqa: E402

CH = 2
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
M32 = 0xFFFFFFFF


# =========================================================================== the demos' generator
def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """DK.rng(seed) of courses/demo-kit.js: mulberry32 uniforms, one call at a time."""

    def __init__(self, seed):
        self.a = int(seed) & M32

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0


def js_uniforms(seed, n):
    """The first n uniforms of DK.rng(seed), vectorised (the state advances by a constant)."""
    j = np.arange(1, n + 1, dtype=np.uint64)
    m = np.uint64(M32)
    a = (np.uint64(int(seed) & M32) + j * np.uint64(0x6D2B79F5)) & m
    t = ((a ^ (a >> np.uint64(15))) * (np.uint64(1) | a)) & m
    t = ((t + (((t ^ (t >> np.uint64(7))) * (np.uint64(61) | t)) & m)) & m) ^ t
    return ((t ^ (t >> np.uint64(14))) & m).astype(np.float64) / 4294967296.0


# =========================================================================== the book's data
def _table_after(name, header):
    with open(common.nps_file(name)) as f:
        lines = f.read().splitlines()
    k = next(i for i, s in enumerate(lines) if s.split() == header)
    return [s.split() for s in lines[k + 1:] if s.strip()]


def nerve():
    """nerve.dat: 799 waiting times (seconds) between pulses along a nerve fibre (Cox and Lewis, 1966)."""
    with open(common.nps_file("nerve.dat")) as f:
        return np.array(f.read().split(), float)


def fiji_magnitudes():
    """fijiquakes.dat: Richter magnitudes of 1000 earthquakes near Fiji (column 'mag')."""
    rows = _table_after("fijiquakes.dat", ["Obs.", "lat", "long", "depth", "mag", "stations"])
    return np.array([float(r[4]) for r in rows])


def faithful():
    """faithful.dat: (eruption durations in minutes, waiting times in minutes), 272 eruptions."""
    rows = _table_after("faithful.dat", ["eruptions", "waiting"])
    a = np.array([[float(r[1]), float(r[2])] for r in rows])
    return a[:, 0], a[:, 1]


def clouds():
    """clouds.dat: rainfall (acre-feet) from 26 unseeded and 26 seeded clouds; returns (unseeded, seeded)."""
    rows = _table_after("clouds.dat", ["Unseeded_Clouds", "Seeded_Clouds"])
    a = np.array([[float(x), float(y)] for x, y in rows])
    return a[:, 0], a[:, 1]


# =========================================================================== Section 2.1
def ecdf(x, t):
    """F_n(t) = #{X_i <= t} / n at the points t (Definition 2.1)."""
    xs = np.sort(np.asarray(x, float))
    return np.searchsorted(xs, np.asarray(t, float), side="right") / xs.size


def dkw_eps(n, alpha):
    """Half-width of the DKW band, eps_n = sqrt(log(2 / alpha) / (2 n)) (Theorem 2.6)."""
    return math.sqrt(math.log(2.0 / alpha) / (2.0 * n))


def vc_eps(n, alpha, v=1):
    """Half-width that solves 8 (n^v + 1) exp(-n eps^2 / 32) = alpha, the bound (2.44)."""
    return math.sqrt(32.0 / n * math.log(8.0 * (float(n) ** v + 1.0) / alpha))


def dkw_band(x, t, alpha=0.05):
    F = ecdf(x, t)
    e = dkw_eps(len(x), alpha)
    return F, np.maximum(F - e, 0.0), np.minimum(F + e, 1.0)


def sup_dist_uniform(u):
    """sup_x |F_n(x) - x| for a sample u of Uniform(0, 1) (attained at the order statistics)."""
    u = np.sort(u)
    n = u.size
    i = np.arange(1, n + 1)
    return max(np.max(i / n - u), np.max(u - (i - 1) / n))


def band_coverage(n=100, R=1000, alpha=0.05, seed=2300):
    """Exercise 2.3 as the demo runs it: sample r is X_i = F^{-1}(U_i) with U_1..U_n the first n
    uniforms of DK.rng(seed + r). For every continuous F the band covers F exactly when
    sup|F_n(u) - u| <= eps_n for the uniforms, so the count is the same for Normal and Cauchy data.
    Returns (number of samples whose band contains F, list of the R distances)."""
    e = dkw_eps(n, alpha)
    D = np.array([sup_dist_uniform(js_uniforms(seed + r, n)) for r in range(R)])
    return int(np.sum(D <= e)), D


def band_coverage_direct(dist, n=100, R=1000, alpha=0.05, seed=23):
    """Exercise 2.3 by brute force: simulate from dist (a scipy frozen law), evaluate F on a fine grid
    and at the data, and check whether L <= F <= U everywhere."""
    rng = np.random.default_rng(seed)
    e = dkw_eps(n, alpha)
    hits = 0
    for _ in range(R):
        x = np.sort(dist.rvs(size=n, random_state=rng))
        F = dist.cdf(x)
        i = np.arange(1, n + 1)
        D = max(np.max(i / n - F), np.max(F - (i - 1) / n))
        hits += D <= e
    return int(hits)


# =========================================================================== Sections 2.2 and 2.3
def wquantile(x, w, p):
    """inf{t : G(t) >= p} for the discrete distribution with atoms x and weights w (Example 2.17)."""
    o = np.argsort(x, kind="mergesort")
    xs, cw = np.asarray(x, float)[o], np.cumsum(np.asarray(w, float)[o])
    k = int(np.searchsorted(cw, p - 1e-12, side="left"))
    return float(xs[min(k, xs.size - 1)])


def functional(name, x, w=None, p=0.5, a=None, b=None):
    """T(G) for the discrete distribution G with atoms x (n values, or n x 2 for 'corr') and weights w
    (default 1/n: then G = F_n and T(G) is the plug-in estimate). name is one of 'mean', 'var',
    'skew', 'corr', 'quantile' (with p) and 'prob' (F(b) - F(a))."""
    x = np.asarray(x, float)
    n = x.shape[0]
    w = np.full(n, 1.0 / n) if w is None else np.asarray(w, float) / np.sum(w)
    if name == "corr":
        mx, my = w @ x[:, 0], w @ x[:, 1]
        sxy = w @ ((x[:, 0] - mx) * (x[:, 1] - my))
        return float(sxy / math.sqrt((w @ (x[:, 0] - mx) ** 2) * (w @ (x[:, 1] - my) ** 2)))
    if name == "quantile":
        return wquantile(x, w, p)
    if name == "prob":
        return float(w @ ((x > a) & (x <= b)))
    m = w @ x
    if name == "mean":
        return float(m)
    s2 = w @ (x - m) ** 2
    if name == "var":
        return float(s2)
    if name == "skew":
        return float((w @ (x - m) ** 3) / s2 ** 1.5)
    raise ValueError(name)


def silverman_h(x):
    x = np.asarray(x, float)
    iqr = np.subtract(*np.percentile(x, [75, 25]))
    return 0.9 * min(np.std(x, ddof=1), iqr / 1.34) * x.size ** (-0.2)


def kde_at(x, t, h=None):
    """Gaussian kernel density estimate at t, bandwidth h (default Silverman's rule of thumb)."""
    x = np.asarray(x, float)
    h = silverman_h(x) if h is None else h
    return float(np.mean(stats.norm.pdf((t - x) / h)) / h)


def influence(name, x, at, p=0.5, a=None, b=None, fhat=None):
    """Empirical influence function L_hat(at) = L_{F_n}(at) in closed form (Theorem 2.22,
    Examples 2.32, 2.34, 2.35, Exercise 2.8). For 'corr', at is an (m, 2) array. For 'quantile' the
    density at the sample quantile is fhat (default: kde_at)."""
    x = np.asarray(x, float)
    at = np.asarray(at, float)
    if name == "corr":
        mx, my = x[:, 0].mean(), x[:, 1].mean()
        sx, sy = x[:, 0].std(), x[:, 1].std()
        r = functional("corr", x)
        u, v = (at[..., 0] - mx) / sx, (at[..., 1] - my) / sy
        return u * v - 0.5 * r * (u ** 2 + v ** 2)
    if name == "quantile":
        th = functional("quantile", x, p=p)
        f = kde_at(x, th) if fhat is None else fhat
        return (p - (at <= th)) / f
    if name == "prob":
        th = functional("prob", x, a=a, b=b)
        return ((at > a) & (at <= b)) - th
    m, s2 = x.mean(), x.var()
    if name == "mean":
        return at - m
    if name == "var":
        return (at - m) ** 2 - s2
    if name == "skew":
        s, m3 = math.sqrt(s2), np.mean((x - m) ** 3)
        k = m3 / s ** 3
        return ((at - m) ** 3 - m3 - 3 * s2 * (at - m)) / s ** 3 - 1.5 * k * ((at - m) ** 2 - s2) / s2
    raise ValueError(name)


def gateaux(name, x, pt, eps=1e-6, **kw):
    """The difference quotient (T((1 - eps) F_n + eps delta_pt) - T(F_n)) / eps of (2.21)."""
    x = np.asarray(x, float)
    n = x.shape[0]
    pt = np.atleast_1d(np.asarray(pt, float))
    xx = np.concatenate([x, pt[None, :] if x.ndim == 2 else pt[:1]])
    w = np.concatenate([np.full(n, (1 - eps) / n), [eps]])
    return (functional(name, xx, w, **kw) - functional(name, x, **kw)) / eps


def delta_se(name, x, **kw):
    """se_hat = tau_hat / sqrt(n), tau_hat^2 = mean of L_hat(X_i)^2 (the nonparametric delta method)."""
    x = np.asarray(x, float)
    L = influence(name, x, x, **kw)
    return float(math.sqrt(np.mean(L ** 2) / x.shape[0]))


def delta_ci(name, x, alpha=0.05, **kw):
    t = functional(name, x, **{k: v for k, v in kw.items() if k != "fhat"})
    se = delta_se(name, x, **kw)
    z = stats.norm.ppf(1 - alpha / 2)
    return t, se, t - z * se, t + z * se


def quantile_if_population(dist, p, x, eps=1e-7):
    """Gateaux derivative of the p-quantile at a continuous F in the direction delta_x, by solving
    (1 - eps) F(t) + eps 1{t >= x} = p for t (the hint of Exercise 2.10)."""
    th = dist.ppf(p)
    if x > th:                              # the mass at x lies above the quantile
        t = dist.ppf(p / (1 - eps))
    else:                                   # x <= theta: solve with the jump included
        t = dist.ppf((p - eps) / (1 - eps))
        if t < x:                           # the jump at x already carries G past p
            t = x
    return (t - th) / eps


# Influence functions at F = N(0, 1) (closed forms used by Figure 2.S1 and the checks)
def if_normal(name, x, p=0.5):
    x = np.asarray(x, float)
    if name == "mean":
        return x
    if name == "var":
        return x ** 2 - 1
    if name == "skew":
        return x ** 3 - 3 * x
    if name == "quantile":
        th = stats.norm.ppf(p)
        return (p - (x <= th)) / stats.norm.pdf(th)
    raise ValueError(name)


def gateaux_population(name, x, eps=1e-6, p=0.5):
    """(T((1 - eps) Phi + eps delta_x) - T(Phi)) / eps at the standard Normal, from exact moments."""
    if name == "quantile":
        return quantile_if_population(stats.norm, p, x, eps)
    m1 = eps * x
    m2 = (1 - eps) + eps * x * x
    m3 = eps * x ** 3
    mu = m1
    var = m2 - mu ** 2
    if name == "mean":
        return m1 / eps
    if name == "var":
        return (var - 1.0) / eps
    if name == "skew":
        c3 = m3 - 3 * mu * m2 + 2 * mu ** 3
        return (c3 / var ** 1.5) / eps
    raise ValueError(name)


# =========================================================================== Section 2.4
CLASSES = ("halfline", "interval", "halfplane", "rectangle", "disc")
VC_DIM = {"halfline": 1, "interval": 2, "halfplane": 3, "rectangle": 4, "disc": 3}


def _mask(idx):
    m = 0
    for i in idx:
        m |= 1 << int(i)
    return m


def _with_boundary(inside, bd):
    base = _mask(inside)
    out = set()
    for r in range(len(bd) + 1):
        for c in itertools.combinations(bd, r):
            out.add(base | _mask(c))
    return out


def picked_sets(P, cls, tol=1e-9):
    """The set of bitmasks R intersect A, A in the class, for the points P (m x 2 array; the 1-D
    classes use the first coordinate). Exact for points in general position:
    halfline (-inf, t]: prefixes of the sorted points; interval [s, t]: runs of consecutive sorted
    points; rectangle: S is picked out iff its bounding box holds no other point; halfplane: every
    picked set is the strict side of a line through two points plus any subset of those two;
    disc: every picked set is the interior of a circle through one, two (as diameter) or three
    points plus any subset of the points on the circle (Section 2.4 of the notes proves both)."""
    P = np.asarray(P, float)
    m = len(P)
    out = {0}
    if m == 0:
        return out
    if cls in ("halfline", "interval"):
        o = np.argsort(P[:, 0], kind="mergesort")
        if cls == "halfline":
            for k in range(1, m + 1):
                out.add(_mask(o[:k]))
        else:
            for i in range(m):
                for j in range(i, m):
                    out.add(_mask(o[i:j + 1]))
        return out
    if cls == "rectangle":
        for S in range(1, 1 << m):
            idx = [i for i in range(m) if S >> i & 1]
            lo, hi = P[idx].min(0), P[idx].max(0)
            inside = np.all((P >= lo - tol) & (P <= hi + tol), axis=1)
            if _mask(np.nonzero(inside)[0]) == S:
                out.add(S)
        return out
    out.add((1 << m) - 1)
    if cls == "halfplane":
        for i, j in itertools.combinations(range(m), 2):
            d = P[j] - P[i]
            nrm = np.array([-d[1], d[0]])
            s = (P - P[i]) @ nrm
            bd = [k for k in range(m) if abs(s[k]) <= tol * (1 + np.abs(nrm).sum())]
            for side in (1, -1):
                out |= _with_boundary([k for k in range(m) if side * s[k] > tol * (1 + np.abs(nrm).sum())], bd)
        return out
    if cls == "disc":
        def add_circle(c, r2):
            d2 = np.sum((P - c) ** 2, axis=1)
            sc = 1e-9 * (1 + r2)
            out.update(_with_boundary([k for k in range(m) if d2[k] < r2 - sc],
                                      [k for k in range(m) if abs(d2[k] - r2) <= sc]))
        for i in range(m):
            out.add(1 << i)
        for i, j in itertools.combinations(range(m), 2):
            add_circle((P[i] + P[j]) / 2, np.sum((P[i] - P[j]) ** 2) / 4)
        for i, j, k in itertools.combinations(range(m), 3):
            a, b, c = P[i], P[j], P[k]
            D = 2 * (a[0] * (b[1] - c[1]) + b[0] * (c[1] - a[1]) + c[0] * (a[1] - b[1]))
            if abs(D) < 1e-12:
                continue
            ux = ((a @ a) * (b[1] - c[1]) + (b @ b) * (c[1] - a[1]) + (c @ c) * (a[1] - b[1])) / D
            uy = ((a @ a) * (c[0] - b[0]) + (b @ b) * (a[0] - c[0]) + (c @ c) * (b[0] - a[0])) / D
            cc = np.array([ux, uy])
            add_circle(cc, np.sum((a - cc) ** 2))
        return out
    raise ValueError(cls)


def shatter_count(P, cls):
    return len(picked_sets(P, cls))


def js_points(seed, m):
    """The demo's random point set: m points with coordinates 0.1 + 0.8 U, U from DK.rng(seed),
    drawn x1, y1, x2, y2, ..."""
    g = JSRandom(seed)
    return np.array([[0.1 + 0.8 * g.u(), 0.1 + 0.8 * g.u()] for _ in range(m)])


PRESET_POINTS = {
    "two": np.array([[0.3, 0.5], [0.7, 0.5]]),
    "three": np.array([[0.3, 0.45], [0.5, 0.6], [0.7, 0.45]]),
    "triangle": np.array([[0.5, 0.8], [0.24, 0.35], [0.76, 0.35]]),
    "diamond": np.array([[0.5, 0.85], [0.15, 0.52], [0.86, 0.47], [0.52, 0.15]]),
    "pentagon": np.array([[0.5, 0.88], [0.14, 0.62], [0.28, 0.18], [0.72, 0.18], [0.86, 0.58]]),
    "kite": np.array([[0.5, 0.88], [0.18, 0.55], [0.5, 0.12], [0.8, 0.5]]),
}


def sauer(n, v):
    """Sauer's bound sum_{i=0}^{v} C(n, i) on the shatter coefficient (Filling the gap 2.10)."""
    return sum(math.comb(n, i) for i in range(v + 1))


def vc_bound(n, eps, s):
    return 8.0 * s * math.exp(-n * eps ** 2 / 32.0)


# =========================================================================== exercises
def exercise_numbers():
    out = {}
    hits, _ = band_coverage(100, 1000, 0.05, 2300)
    out["ex3_js"] = hits
    out["ex3_normal"] = band_coverage_direct(stats.norm(), seed=23)
    out["ex3_cauchy"] = band_coverage_direct(stats.cauchy(), seed=24)
    # Exercise 2.11: Fiji magnitudes, F(4.9) - F(4.3)
    mag = fiji_magnitudes()
    out["ex11"] = delta_ci("prob", mag, 0.05, a=4.3, b=4.9)
    out["ex11_eps"] = dkw_eps(mag.size, 0.05)
    # Exercise 2.12: Old Faithful waiting times
    _, wait = faithful()
    t, se, _, _ = delta_ci("mean", wait)
    z90 = stats.norm.ppf(0.95)
    out["ex12"] = dict(mean=t, se=se, ci90=(t - z90 * se, t + z90 * se),
                       median=functional("quantile", wait, p=0.5))
    # Exercise 2.13: cloud seeding, difference of medians (seeded minus unseeded)
    un, se_ = clouds()
    res = {}
    for lab, a in (("seeded", se_), ("unseeded", un)):
        m = functional("quantile", a, p=0.5)
        f_raw = kde_at(a, m)                                   # kernel estimate on the raw scale
        la = np.log(a)
        f_log = kde_at(la, math.log(m)) / m                    # change of variables from the log scale
        res[lab] = dict(median=m, f_raw=f_raw, f_log=f_log,
                        se_raw=math.sqrt(0.25 / (a.size * f_raw ** 2)),
                        se_log=math.sqrt(0.25 / (a.size * f_log ** 2)))
    th = res["seeded"]["median"] - res["unseeded"]["median"]
    for k in ("raw", "log"):
        se_d = math.hypot(res["seeded"]["se_" + k], res["unseeded"]["se_" + k])
        res["diff_" + k] = (th, se_d, th - 1.96 * se_d, th + 1.96 * se_d)
    # a third answer: the bootstrap (Chapter 3), 4000 resamples
    rng = np.random.default_rng(213)
    B = 4000
    med = lambda a: functional("quantile", a, p=0.5)
    bs = np.array([med(rng.choice(se_, se_.size)) - med(rng.choice(un, un.size)) for _ in range(B)])
    res["boot_se"] = float(bs.std(ddof=1))
    out["ex13"] = res
    return out


def export_json():
    os.makedirs(DATA_OUT, exist_ok=True)
    er, wait = faithful()
    un, se_ = clouds()
    d = dict(nerve=[float(v) for v in nerve()], fiji=[float(v) for v in fiji_magnitudes()],
             eruptions=[float(v) for v in er], waiting=[float(v) for v in wait],
             clouds_unseeded=[float(v) for v in un], clouds_seeded=[float(v) for v in se_])
    path = os.path.join(DATA_OUT, "ch02-data.json")
    with open(path, "w") as f:
        json.dump(d, f, separators=(",", ":"))
    print("wrote", os.path.relpath(path, os.getcwd()))


# =========================================================================== figures
def _rug(ax, x, y0, h, color="0.35"):
    u = np.unique(x)
    ax.vlines(u, y0, y0 + h, color=color, lw=0.5)


def fig_2_1(out_dir=None):
    """Figure 2.1: nerve data, empirical cdf and 95 percent DKW confidence band."""
    x = nerve()
    t = np.linspace(-0.02, 1.5, 3041)
    F, L, U = dkw_band(x, t, 0.05)
    fig, ax = common.plt.subplots(figsize=(5.4, 3.6))
    ax.plot(t, F, color="k", lw=1.5, drawstyle="steps-post", label=r"empirical cdf $\widehat F_n$")
    ax.plot(t, U, color=RED, lw=1.1, ls="--", drawstyle="steps-post", label="95% DKW band")
    ax.plot(t, L, color=RED, lw=1.1, ls="--", drawstyle="steps-post")
    _rug(ax, x, -0.02, 0.06)
    ax.set_xlim(-0.02, 1.5)
    ax.set_ylim(-0.03, 1.05)
    ax.set_xlabel("waiting time between pulses (seconds)")
    ax.set_ylabel("cumulative probability")
    ax.legend(loc="center right", frameon=False)
    ax.text(0.72, 0.42, r"$n=799,\ \epsilon_n=%.4f$" % dkw_eps(x.size, 0.05), fontsize=10)
    return savefig(fig, CH, "fig2-1", out_dir)


def fig_2_s1(out_dir=None):
    """Figure 2.S1: influence functions at N(0, 1), closed form and difference quotients."""
    xs = np.linspace(-3, 3, 601)
    pts = np.linspace(-2.75, 2.75, 12)
    fig, axes = common.panel_grid(1, 4, size=(2.55, 2.4))
    specs = [("mean", {}, "mean"), ("var", {}, "variance"), ("skew", {}, "skewness"),
             ("quantile", {"p": 0.5}, "median")]
    for ax, (name, kw, lab) in zip(axes[0], specs):
        y = if_normal(name, xs, **kw)
        ax.plot(xs, y, color=BLUE, lw=1.6, drawstyle="steps-mid" if name == "quantile" else "default")
        g = [gateaux_population(name, p0, **kw) for p0 in pts]
        ax.plot(pts, g, "o", ms=3.6, color=RED, mfc="none", mew=1.1)
        ax.axhline(0, color="0.6", lw=0.6)
        ax.set_title(lab)
        ax.set_xlabel("$x$")
    axes[0, 0].set_ylabel(r"$L_F(x)$")
    axes[0, 0].plot([], [], color=BLUE, label="closed form")
    axes[0, 0].plot([], [], "o", color=RED, mfc="none", label=r"quotient, $\epsilon=10^{-6}$")
    axes[0, 0].legend(loc="upper left", frameon=False, fontsize=7.5)
    fig.tight_layout()
    return savefig(fig, CH, "fig2-S1", out_dir)


def fig_2_s2(out_dir=None):
    """Figure 2.S2: Exercise 2.11, Fiji earthquake magnitudes with a 95 percent DKW band."""
    mag = fiji_magnitudes()
    t = np.linspace(3.9, 6.5, 2601)
    F, L, U = dkw_band(mag, t, 0.05)
    fig, ax = common.plt.subplots(figsize=(5.4, 3.4))
    ax.axvspan(4.3, 4.9, color=GREEN, alpha=0.12, lw=0)
    ax.plot(t, F, color="k", lw=1.5, drawstyle="steps-post", label=r"$\widehat F_n$")
    ax.plot(t, U, color=RED, lw=1.1, ls="--", drawstyle="steps-post", label="95% DKW band")
    ax.plot(t, L, color=RED, lw=1.1, ls="--", drawstyle="steps-post")
    ax.set_xlim(3.9, 6.5)
    ax.set_ylim(-0.02, 1.03)
    ax.set_xlabel("magnitude (Richter scale)")
    ax.set_ylabel("cumulative probability")
    ax.text(4.32, 0.07, r"$(4.3,\,4.9]$", color=GREEN, fontsize=9)
    ax.legend(loc="lower right", frameon=False)
    return savefig(fig, CH, "fig2-S2", out_dir)


def delta_sim(name, R=4000, n=100, seed=2033):
    """Studentised plug-in estimates (T(F_n) - T(F)) / se_hat over R samples of size n."""
    rng = np.random.default_rng(seed)
    out = np.empty(R)
    for r in range(R):
        if name == "corr":
            rho = 0.6
            z = rng.standard_normal((n, 2))
            x = np.column_stack([z[:, 0], rho * z[:, 0] + math.sqrt(1 - rho ** 2) * z[:, 1]])
            truth = rho
        elif name == "quantile":
            x = rng.exponential(1.0, n)
            truth = math.log(2.0)
        elif name == "skew":
            x = rng.exponential(1.0, n)
            truth = 2.0
        else:
            x = rng.exponential(1.0, n)
            truth = 1.0
        t = functional(name, x)
        out[r] = (t - truth) / delta_se(name, x)
    return out


def fig_2_s3(out_dir=None):
    """Figure 2.S3: the nonparametric delta method, studentised estimates against N(0, 1)."""
    specs = [("var", "variance, Exp(1)"), ("quantile", "median, Exp(1)"),
             ("corr", r"correlation, Normal $\rho=0.6$"), ("skew", "skewness, Exp(1)")]
    fig, axes = common.panel_grid(1, 4, size=(2.55, 2.3))
    z = np.linspace(-4.5, 4.5, 400)
    for ax, (name, lab) in zip(axes[0], specs):
        s = delta_sim(name)
        ax.hist(np.clip(s, -6, 6), bins=np.linspace(-6, 6, 49), density=True, color=BLUE, alpha=0.45)
        ax.plot(z, stats.norm.pdf(z), color="k", lw=1.2)
        cov = np.mean(np.abs(s) <= 1.96)
        ax.set_title(lab, fontsize=9)
        ax.text(0.03, 0.95, "95%% CI covers\n%.3f" % cov, transform=ax.transAxes, va="top", fontsize=8)
        ax.set_xlim(-6, 6)
        ax.set_xlabel(r"$(T(\widehat F_n)-T(F))/\widehat{\rm se}$", fontsize=8)
    fig.tight_layout()
    return savefig(fig, CH, "fig2-S3", out_dir)


def shatter_curves(cls, nmax=8, seeds=range(40)):
    """max over random point sets (DK.rng seeds) of the number of picked-out subsets, n = 1..nmax."""
    return [max(shatter_count(js_points(1000 + s, n), cls) for s in seeds) for n in range(1, nmax + 1)]


def fig_2_s4(out_dir=None):
    """Figure 2.S4: shatter coefficients against Sauer's bound, and the DKW against the VC band."""
    fig, axes = common.panel_grid(1, 2, size=(3.6, 2.9))
    ax = axes[0, 0]
    ns = np.arange(1, 9)
    cols = {"halfline": GREY, "interval": GREEN, "halfplane": BLUE, "rectangle": PURPLE, "disc": ORANGE}
    for cls in CLASSES:
        c = shatter_curves(cls)
        ax.plot(ns, c, "o-", ms=3.5, color=cols[cls], lw=1.2, label="%s (v=%d)" % (cls, VC_DIM[cls]))
        ax.plot(ns, [sauer(n, VC_DIM[cls]) for n in ns], ":", color=cols[cls], lw=1)
    ax.plot(ns, 2.0 ** ns, color="k", lw=1, label=r"$2^n$")
    ax.set_yscale("log")
    ax.set_xlabel("number of points $n$")
    ax.set_ylabel("subsets picked out")
    ax.legend(frameon=False, fontsize=6.8, loc="upper left")
    ax = axes[0, 1]
    n = np.unique(np.round(np.logspace(1, 6, 60)).astype(int))
    ax.loglog(n, [dkw_eps(k, 0.05) for k in n], color=RED, label="DKW (2.5)")
    ax.loglog(n, [vc_eps(k, 0.05, 1) for k in n], color=BLUE, label="VC (2.44), $v=1$")
    ax.axhline(1, color="0.6", lw=0.7, ls="--")
    ax.set_xlabel("sample size $n$")
    ax.set_ylabel(r"95% band half-width $\epsilon_n$")
    ax.legend(frameon=False, fontsize=8)
    fig.tight_layout()
    return savefig(fig, CH, "fig2-S4", out_dir)


FIGURES = {"2.1": fig_2_1, "2.S1": fig_2_s1, "2.S2": fig_2_s2, "2.S3": fig_2_s3, "2.S4": fig_2_s4}


if __name__ == "__main__":
    if "--json" in sys.argv:
        export_json()
    elif "--exercises" in sys.argv:
        import pprint
        pprint.pprint(exercise_numbers())
    else:
        export_json()
        common.run(FIGURES, CH)
