"""Wasserman (2004), Chapter 6: Models, Statistical Inference and Learning.

Companion code for Chapter 6 of L. Wasserman, All of Statistics: A Concise Course in Statistical
Inference (Springer, 2004), pp. 87-96, written from scratch with numpy, scipy and matplotlib. The
functions are short and meant to be read next to the lecture notes (../ch06-models-inference-learning.html):

* Section 6.2, models: bump_pair (two densities that are close in total variation but differ by a
  fixed amount at a point) and lecam_bound (the two-point lower bound that shows why a density
  cannot be estimated at a point over the class of all densities, Example 6.4).
* Section 6.3.1, point estimation: exact bias, variance and mean squared error (6.4)-(6.7) of the
  shrinkage estimators of the browser demonstration (shrink_moments: c * mean for a Normal mean,
  (S + a) / (n + 2a) for a Bernoulli probability, c * max for a Uniform(0, theta) endpoint),
  their Monte Carlo counterparts (shrink_mc), and the exact probabilities P(|theta_n - theta| > eps)
  of five estimators, consistent or not (tail_prob), next to the Chebyshev and Hoeffding bounds.
* Section 6.3.2, confidence sets: the Wald (Normal-based, Theorem 6.16 and Example 6.17), Hoeffding
  (Example 6.15) and Wilson intervals for a binomial proportion (binom_interval) and their exact
  coverage and expected width (binom_coverage), the frequentist coverage of a Bayesian posterior
  interval (bayes_coverage), and the example of Berger and Wolpert (berger_wolpert).
* Section 6.3.3, hypothesis testing: the exact size of the Normal-approximation test of a fair coin
  (fair_coin_size, Example 6.18).
* Exercises 6.1-6.3: uniform_estimators (bias, standard error and MSE of 2 * mean, max and its
  rescaled versions).
* JSRandom and js_uniforms / js_normals: the seeded generator of the browser demonstrations
  (DK.rng in courses/demo-kit.js), so that the Monte Carlo numbers on the pages can be reproduced
  here number for number.

Chapter 6 of the book has no figures (it is a chapter of definitions and small examples), so
nothing is reproduced and nothing is skipped. The module draws six supplementary figures of our own
that illustrate the gaps filled in the notes:
  6.S1 two densities that are indistinguishable from samples but differ at a point (Example 6.4);
  6.S2 MSE = bias^2 + variance for shrinkage estimators (Theorem 6.9);
  6.S3 consistency as n grows, and an estimator that is not consistent (Definition 6.7, Theorem 6.10);
  6.S4 exact coverage of three 95 percent intervals for a proportion (Examples 6.15 and 6.17, Section 6.5);
  6.S5 what "95 percent" means: unrelated parameters, Example 6.14, and Bayesian intervals;
  6.S6 the estimators of Exercises 6.2 and 6.3.
Run `python3 ch06_models_inference_learning.py --list`, `python3 ch06_models_inference_learning.py`
(all six figures, about 20 seconds on one CPU thread) or `python3 ch06_models_inference_learning.py --fig 6.S4`.
Files go to ../img/ch06/. The three demonstrations simulate their own data and need no data files.
"""
import math
import os

import numpy as np
from scipy import special, stats

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, savefig

CH = 6
M32 = 0xFFFFFFFF


# =========================================================================== the demos' generator
def _imul(a, b):
    """32-bit integer multiplication with wrap-around, as Math.imul (result as unsigned)."""
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js), one call at a
    time: u() is mulberry32 and norm() the Box-Muller transform that returns r cos(theta) and keeps
    r sin(theta) for the next call, exactly as in JavaScript."""

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        x = 0.0
        while x == 0.0:
            x = self.u()
        r = math.sqrt(-2.0 * math.log(x))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


def js_uniforms(seed, n, start=0):
    """Uniforms number start+1, ..., start+n of DK.rng(seed), vectorised: mulberry32 adds a constant
    to its state at each call, so the j-th output is a hash of seed + j * 0x6D2B79F5 (mod 2^32)."""
    j = np.arange(start + 1, start + n + 1, dtype=np.uint64)
    m = np.uint64(M32)
    a = (np.uint64(int(seed) & M32) + j * np.uint64(0x6D2B79F5)) & m
    t = ((a ^ (a >> np.uint64(15))) * (np.uint64(1) | a)) & m
    t = ((t + (((t ^ (t >> np.uint64(7))) * (np.uint64(61) | t)) & m)) & m) ^ t
    return ((t ^ (t >> np.uint64(14))) & m).astype(np.float64) / 4294967296.0


def js_normals(seed, n, start=0):
    """n consecutive DK.rng(seed).norm() values drawn after `start` uniforms with no normal cached
    (Box-Muller pairs: cos first, then the cached sin)."""
    npair = (n + 1) // 2
    u = js_uniforms(seed, 2 * npair, start)
    u1, u2 = u[0::2], u[1::2]
    if np.any(u1 == 0.0):
        raise RuntimeError("a zero uniform: use JSRandom for this stream")
    r = np.sqrt(-2.0 * np.log(u1))
    th = 2.0 * np.pi * u2
    out = np.empty(2 * npair)
    out[0::2] = r * np.cos(th)
    out[1::2] = r * np.sin(th)
    return out[:n]


def zq(alpha):
    """z_{alpha/2} = Phi^{-1}(1 - alpha/2)."""
    return float(stats.norm.ppf(1.0 - alpha / 2.0))


# =========================================================================== Section 6.2: Example 6.4
def bump(u):
    """The smooth bump psi(u) = exp(1 - 1/(1 - u^2)) on |u| < 1 (psi(0) = 1, zero outside)."""
    u = np.asarray(u, dtype=float)
    out = np.zeros_like(u)
    inside = np.abs(u) < 1
    out[inside] = np.exp(1.0 - 1.0 / (1.0 - u[inside] ** 2))
    return out


def _bump_integral():
    from scipy.integrate import quad
    return quad(lambda u: math.exp(1.0 - 1.0 / (1.0 - u * u)) if abs(u) < 1 else 0.0, -1, 1, epsabs=1e-13)[0]


BUMP_INTEGRAL = _bump_integral()       # integral of psi over (-1, 1), about 1.2069


def bump_pair(x, delta, h, x0=0.0, x1=1.0):
    """Densities f = N(0, 1) and g = f + delta psi((x - x0)/h) - delta psi((x - x1)/h).

    g integrates to 1, is nonnegative when delta <= f on (x1 - h, x1 + h) (true for delta = 0.2,
    x1 = 1 and h <= 0.15, since f >= 0.206 there), has g(x0) = f(x0) + delta, and
    TV(f, g) = (1/2) int |g - f| = delta h BUMP_INTEGRAL (the two bumps do not overlap)."""
    f = stats.norm.pdf(x)
    g = f + delta * bump((x - x0) / h) - delta * bump((x - x1) / h)
    return f, g


def lecam_bound(n, tv):
    """Le Cam's two-point bound: for any estimator T of f(x0), from n observations,
    max over {f, g} of P(|T - f(x0)| >= delta/2) >= (1 - TV(f^n, g^n)) / 2 >= (1 - n tv) / 2."""
    return np.maximum(0.0, 1.0 - np.asarray(n, dtype=float) * tv) / 2.0


def sobolev_penalty(delta, h):
    """int (g'' - f'')^2 dx for the bump pair: the two bumps each contribute
    delta^2 h^{-3} int psi''(u)^2 du, so the Sobolev seminorm explodes as h -> 0."""
    from scipy.integrate import quad

    def d2(u):
        if abs(u) >= 1:
            return 0.0
        s = 1.0 - u * u
        p = math.exp(1.0 - 1.0 / s)
        # psi' = psi * (-2u / s^2); psi'' = psi * ((2u/s^2)^2 - (2/s^2 + 8u^2/s^3))
        return p * ((2 * u / s ** 2) ** 2 - (2.0 / s ** 2 + 8.0 * u * u / s ** 3))
    i2 = quad(lambda u: d2(u) ** 2, -1, 1, limit=200)[0]
    return 2.0 * delta ** 2 * i2 / h ** 3


# =========================================================================== Section 6.3.1: point estimation
def shrink_moments(model, n, theta, t, sigma=1.0):
    """Exact mean, bias, variance and MSE (6.4)-(6.7) of the estimators of the MSE demonstration.

    model "normal":  X_i ~ N(theta, sigma^2), estimator t * mean (shrinkage towards 0, t = c);
    model "bern":    X_i ~ Bernoulli(theta), estimator (S + t) / (n + 2t) (t = a pseudo-counts);
    model "unif":    X_i ~ Uniform(0, theta), estimator t * max (t = c)."""
    if model == "normal":
        m = t * theta
        v = t * t * sigma ** 2 / n
    elif model == "bern":
        m = (n * theta + t) / (n + 2.0 * t)
        v = n * theta * (1 - theta) / (n + 2.0 * t) ** 2
    elif model == "unif":
        m = t * n * theta / (n + 1.0)
        v = t * t * n * theta ** 2 / ((n + 1.0) ** 2 * (n + 2.0))
    else:
        raise ValueError(model)
    b = m - theta
    return dict(mean=m, bias=b, var=v, mse=b * b + v)


def shrink_best(model, n, theta, sigma=1.0):
    """The tuning value that minimises the MSE (it depends on the unknown theta, except for unif)."""
    if model == "normal":
        return theta ** 2 / (theta ** 2 + sigma ** 2 / n)
    if model == "unif":
        return (n + 2.0) / (n + 1.0)
    if model == "bern":
        # for the add-a estimator the minimiser is a* = p (1 - p) / (2 (p - 1/2)^2) (infinite at p = 1/2);
        # found numerically here, so that the closed form can be compared with it
        from scipy.optimize import minimize_scalar
        r = minimize_scalar(lambda a: shrink_moments("bern", n, theta, a)["mse"], bounds=(0, 50 * n), method="bounded",
                            options=dict(xatol=1e-10))
        return float(r.x)
    raise ValueError(model)


def shrink_mc(model, n, theta, t, R, seed, sigma=1.0):
    """R Monte Carlo replicates of the estimator with DK.rng(seed), in the order of the demo:
    replicate r draws its n observations one after the other (normal: n normals; bern: n uniforms,
    X = 1 if u < theta; unif: n uniforms, X = theta u)."""
    if model == "normal":
        z = js_normals(seed, R * n).reshape(R, n) if n % 2 == 0 else None
        if z is None:          # odd n: a normal is carried over between replicates, as in JavaScript
            g = JSRandom(seed)
            z = np.array([[g.norm() for _ in range(n)] for _ in range(R)])
        est = t * (theta + sigma * z).mean(axis=1)
    else:
        u = js_uniforms(seed, R * n).reshape(R, n)
        if model == "bern":
            est = ((u < theta).sum(axis=1) + t) / (n + 2.0 * t)
        else:
            est = t * theta * u.max(axis=1)
    b = est.mean() - theta
    v = est.var()               # divisor R, as in the demo
    return dict(est=est, mean=est.mean(), bias=b, var=v, mse=float(np.mean((est - theta) ** 2)))


# ================================================================= consistency (Definition 6.7)
CONS_MODELS = {
    "bern": "mean of Bernoulli(0.3) data",
    "nvar": "variance estimate (divisor n) of N(0, 1) data",
    "umax": "maximum of Uniform(0, 1) data",
    "cmean": "mean of standard Cauchy data",
    "cmed": "median of standard Cauchy data",
}
CONS_THETA = {"bern": 0.3, "nvar": 1.0, "umax": 1.0, "cmean": 0.0, "cmed": 0.0}


def tail_prob(model, n, eps):
    """Exact P(|theta_n - theta| > eps) for the five estimators of the consistency demo.
    The Cauchy median is the lower median X_(ceil(n/2)), whose cdf is I_{F(x)}(k, n - k + 1)."""
    n = int(n)
    if model == "bern":
        p = CONS_THETA["bern"]
        k = np.arange(n + 1)
        far = np.abs(k / n - p) > eps
        return float(stats.binom.pmf(k[far], n, p).sum())
    if model == "nvar":
        # n s_n^2 ~ chi^2_{n-1}; |s_n^2 - 1| > eps iff chi2 > n (1 + eps) or chi2 < n (1 - eps)
        if n == 1:
            return 1.0 if eps < 1 else 0.0          # s_1^2 = 0
        lo = stats.chi2.cdf(n * (1 - eps), n - 1) if eps < 1 else 0.0
        return float(stats.chi2.sf(n * (1 + eps), n - 1) + lo)
    if model == "umax":
        return float((1 - eps) ** n) if eps < 1 else 0.0
    if model == "cmean":
        return float(1 - 2 / math.pi * math.atan(eps))    # the mean of n Cauchy is Cauchy
    if model == "cmed":
        k = (n + 1) // 2
        Fp, Fm = 0.5 + math.atan(eps) / math.pi, 0.5 - math.atan(eps) / math.pi
        return float(1 - special.betainc(k, n - k + 1, Fp) + special.betainc(k, n - k + 1, Fm))
    raise ValueError(model)


def cons_mse(model, n):
    """The MSE of the estimator (inf for the Cauchy estimators: the mean has no mean, and the
    median of a sample of size 1 or 2 has infinite variance; we report inf for all n for simplicity)."""
    if model == "bern":
        p = CONS_THETA["bern"]
        return p * (1 - p) / n
    if model == "nvar":
        return 1.0 / n ** 2 + 2.0 * (n - 1) / n ** 2
    if model == "umax":
        return 2.0 / ((n + 1.0) * (n + 2.0))
    return math.inf


def cons_paths(model, K, nmax, seed, grid=None):
    """K sample paths n -> theta_n, evaluated at the sample sizes in grid (default 1..nmax).
    Path j uses DK.rng(seed + 7919 j): uniforms for bern, umax, cmean and cmed (Cauchy X = tan(pi (u - 1/2))),
    normals for nvar. Returns an array of shape (K, len(grid))."""
    grid = np.arange(1, nmax + 1) if grid is None else np.asarray(grid)
    nn = np.arange(1, nmax + 1)
    paths = np.empty((K, len(grid)))
    for j in range(K):
        s = seed + 7919 * j
        if model == "nvar":
            x = js_normals(s, nmax + (nmax % 2))[:nmax]
            c1, c2 = np.cumsum(x), np.cumsum(x * x)
            paths[j] = (c2 / nn - (c1 / nn) ** 2)[grid - 1]
            continue
        u = js_uniforms(s, nmax)
        if model == "bern":
            paths[j] = (np.cumsum(u < CONS_THETA["bern"]) / nn)[grid - 1]
        elif model == "umax":
            paths[j] = np.maximum.accumulate(u)[grid - 1]
        else:
            x = np.tan(math.pi * (u - 0.5))
            if model == "cmean":
                paths[j] = (np.cumsum(x) / nn)[grid - 1]
            else:
                paths[j] = [np.partition(x[:m], (m + 1) // 2 - 1)[(m + 1) // 2 - 1] for m in grid]
    return paths


def cons_mc(model, K, nmax, seed, eps):
    """Fraction of the K paths with |theta_n - theta| > eps at each grid size (the demo's dots)."""
    grid = cons_grid(nmax)
    P = cons_paths(model, K, nmax, seed, grid)
    return grid, np.mean(np.abs(P - CONS_THETA[model]) > eps, axis=0)


def cons_grid(nmax):
    """The sample sizes at which the demo evaluates the paths: 1..20, then about 25 per decade."""
    g = set(range(1, min(nmax, 20) + 1))
    v = 20.0
    while v < nmax:
        v *= 10 ** (1 / 25)
        g.add(min(nmax, int(round(v))))
    g.add(nmax)
    return np.array(sorted(g))


# =========================================================================== Section 6.3.2: intervals for a proportion
def binom_interval(k, n, method, alpha=0.05):
    """Endpoints (lo, hi) of the interval after k successes in n trials.

    wald:      p_hat +- z sqrt(p_hat (1 - p_hat) / n)   (Theorem 6.16, Example 6.17)
    hoeffding: p_hat +- sqrt(log(2/alpha) / (2n))        (Example 6.15)
    wilson:    the set of p with |p_hat - p| <= z sqrt(p (1 - p) / n) (Wilson 1927)."""
    k = np.asarray(k, dtype=float)
    ph = k / n
    z = zq(alpha)
    if method == "wald":
        h = z * np.sqrt(ph * (1 - ph) / n)
        return ph - h, ph + h
    if method == "hoeffding":
        h = math.sqrt(math.log(2 / alpha) / (2 * n))
        return ph - h, ph + h
    if method == "wilson":
        c = 1 + z * z / n
        mid = (ph + z * z / (2 * n)) / c
        h = z * np.sqrt(ph * (1 - ph) / n + z * z / (4 * n * n)) / c
        return mid - h, mid + h
    raise ValueError(method)


def binom_coverage(n, p, method, alpha=0.05):
    """Exact coverage P_p(lo < p < hi) (open interval, as in the book) and expected width of the
    interval clipped to [0, 1], for a scalar p."""
    k = np.arange(n + 1)
    lo, hi = binom_interval(k, n, method, alpha)
    w = stats.binom.pmf(k, n, p)
    cov = float(w[(lo < p) & (p < hi)].sum())
    width = float((w * (np.minimum(hi, 1) - np.maximum(lo, 0))).sum())
    return cov, width


def coverage_curve(n, ps, method, alpha=0.05):
    """Exact coverage at each p in ps (vectorised over p)."""
    k = np.arange(n + 1)
    lo, hi = binom_interval(k, n, method, alpha)
    ps = np.asarray(ps, dtype=float)
    W = stats.binom.pmf(k[None, :], n, ps[:, None])
    inside = (lo[None, :] < ps[:, None]) & (ps[:, None] < hi[None, :])
    return (W * inside).sum(axis=1)


def polls(n, p, R, seed):
    """R simulated polls with DK.rng(seed): poll r uses uniforms r n + 1, ..., (r + 1) n; X = 1 if u < p."""
    u = js_uniforms(seed, R * n).reshape(R, n)
    return (u < p).sum(axis=1)


# ================================================================= Example 6.14 and Bayesian intervals
def berger_wolpert(theta):
    """Enumerate the four equally likely outcomes of Example 6.14 and return the coverage of C,
    overall and conditional on Y1 = Y2 and on Y1 != Y2."""
    hit_eq, hit_ne, n_eq, n_ne = 0, 0, 0, 0
    for x1 in (-1, 1):
        for x2 in (-1, 1):
            y1, y2 = theta + x1, theta + x2
            C = y1 - 1 if y1 == y2 else (y1 + y2) / 2
            hit = C == theta
            if y1 == y2:
                n_eq += 1
                hit_eq += hit
            else:
                n_ne += 1
                hit_ne += hit
    return dict(overall=(hit_eq + hit_ne) / 4, given_equal=hit_eq / n_eq, given_unequal=hit_ne / n_ne)


def bayes_coverage(theta, n, sigma=1.0, tau=1.0, alpha=0.05):
    """Frequentist coverage at theta of the 1 - alpha posterior interval for a Normal mean with the
    prior theta ~ N(0, tau^2): w = tau^2 / (tau^2 + s^2), s^2 = sigma^2 / n, interval
    w xbar +- z sqrt(w) s. Coverage = Phi(z/sqrt(w) + d) - Phi(-z/sqrt(w) + d), d = (1 - w) theta / (w s)."""
    s = sigma / math.sqrt(n)
    w = tau ** 2 / (tau ** 2 + s ** 2)
    z = zq(alpha)
    d = (1 - w) * np.asarray(theta, dtype=float) / (w * s)
    return stats.norm.cdf(z / math.sqrt(w) + d) - stats.norm.cdf(-z / math.sqrt(w) + d)


# =========================================================================== Section 6.3.3: testing a coin
def fair_coin_size(n, alpha=0.05):
    """Exact size of the test that rejects H0: p = 1/2 when |p_hat - 1/2| > z_{alpha/2} / (2 sqrt(n))."""
    k = np.arange(n + 1)
    reject = np.abs(k / n - 0.5) > zq(alpha) / (2 * math.sqrt(n))
    return float(stats.binom.pmf(k[reject], n, 0.5).sum())


# =========================================================================== Exercises 6.1-6.3
def uniform_estimators(n, theta=1.0):
    """Bias, se and MSE of four estimators of theta from Uniform(0, theta) data (Exercises 6.2, 6.3)."""
    out = {}
    out["2 mean"] = dict(bias=0.0, se=theta / math.sqrt(3 * n), mse=theta ** 2 / (3 * n))
    for name, c in (("max", 1.0), ("(n+1)/n max", (n + 1.0) / n), ("(n+2)/(n+1) max", (n + 2.0) / (n + 1.0))):
        m = shrink_moments("unif", n, theta, c)
        out[name] = dict(bias=m["bias"], se=math.sqrt(m["var"]), mse=m["mse"])
    return out


# =========================================================================== figures
def fig_6_s1(out_dir=None):
    """Example 6.4: two densities that samples cannot tell apart but that differ at a point."""
    delta = 0.2
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    ax = axes[0, 0]
    x = np.linspace(-3.5, 3.5, 7001)
    f = stats.norm.pdf(x)
    ax.plot(x, f, color=BLUE, lw=1.8, label="f = N(0, 1)")
    for h, col in ((0.15, ORANGE), (0.03, RED)):
        g = bump_pair(x, delta, h)[1]
        ax.plot(x, g, color=col, lw=1.0, label="g, h = %.2f" % h)
    ax.set_xlabel("x")
    ax.set_ylabel("density")
    ax.set_title("f and g differ by %.1f at x = 0" % delta)
    ax.set_ylim(0, 0.75)
    ax.legend(loc="upper right", fontsize=7.5, frameon=False)
    common.panel_label(ax, "a")

    ax = axes[0, 1]
    ns = np.unique(np.round(np.logspace(0, 4, 200)).astype(int))
    for h, col in ((0.15, ORANGE), (0.03, RED), (0.003, MAGENTA)):
        tv = delta * h * BUMP_INTEGRAL
        ax.plot(ns, lecam_bound(ns, tv), color=col, label="h = %g (TV = %.4f)" % (h, tv))
    ax.set_xscale("log")
    ax.set_ylim(0, 0.52)
    ax.set_xlabel("sample size n")
    ax.set_ylabel("lower bound")
    ax.set_title("Le Cam's two-point bound")
    ax.legend(loc="lower left", fontsize=7.5, frameon=False)
    common.panel_label(ax, "b", loc="upper right")

    ax = axes[0, 2]
    hs = np.logspace(-3, np.log10(0.15), 50)
    pen = [sobolev_penalty(delta, h) for h in hs]
    ax.loglog(hs, pen, color=PURPLE)
    ax.set_xlabel("bump width h")
    ax.set_ylabel(r"$\int (g''-f'')^2\,dx$")
    ax.set_title(r"Sobolev size grows like $h^{-3}$")
    common.panel_label(ax, "c", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig6-S1", out_dir)


def fig_6_s2(out_dir=None):
    """Theorem 6.9: MSE = bias^2 + variance for shrinkage estimators."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    n, theta = 10, 0.5
    ax = axes[0, 0]
    cs = np.linspace(0, 1.3, 261)
    mom = [shrink_moments("normal", n, theta, c) for c in cs]
    ax.plot(cs, [m["bias"] ** 2 for m in mom], color=ORANGE, label=r"bias$^2$")
    ax.plot(cs, [m["var"] for m in mom], color=BLUE, label="variance")
    ax.plot(cs, [m["mse"] for m in mom], color=RED, lw=2.2, label="MSE")
    cmc = np.linspace(0.1, 1.3, 7)
    mc = [shrink_mc("normal", n, theta, c, 4000, 61 + i)["mse"] for i, c in enumerate(cmc)]
    ax.plot(cmc, mc, "o", ms=4.5, mfc="white", mec="k", label="Monte Carlo MSE")
    cb = shrink_best("normal", n, theta)
    ax.axvline(cb, color=GREY, ls=":", lw=1)
    ax.set_xlabel(r"shrinkage factor $c$ in $c\,\bar X_n$")
    ax.set_title(r"Normal mean, $n=10$, $\theta=0.5$")
    ax.legend(fontsize=7.5, frameon=False, loc="upper center")
    common.panel_label(ax, "a", loc="upper right")

    ax = axes[0, 1]
    th = np.linspace(-2, 2, 401)
    for c, col, lab in ((1.0, BLUE, r"$c=1$ (unbiased)"), (0.8, ORANGE, r"$c=0.8$"), (0.5, RED, r"$c=0.5$")):
        ax.plot(th, [shrink_moments("normal", n, t, c)["mse"] for t in th], color=col, label=lab)
    ax.set_ylim(0, 0.35)
    ax.set_xlabel(r"true mean $\theta$")
    ax.set_ylabel("MSE")
    ax.set_title("No estimator wins for every $\\theta$")
    ax.legend(fontsize=7.5, frameon=False, loc="upper center")
    common.panel_label(ax, "b", loc="upper left")

    ax = axes[0, 2]
    n = 20
    ps = np.linspace(0, 1, 401)
    for a, col, lab in ((0.0, BLUE, r"$a=0$ (MLE)"), (1.0, GREEN, r"$a=1$"), (math.sqrt(n) / 2, RED, r"$a=\sqrt{n}/2$")):
        ax.plot(ps, [shrink_moments("bern", n, p, a)["mse"] for p in ps], color=col, label=lab)
    ax.set_xlabel(r"true probability $p$")
    ax.set_title(r"$(S+a)/(n+2a)$, $n=20$")
    ax.legend(fontsize=7.5, frameon=False, loc="upper right")
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig6-S2", out_dir)


def fig_6_s3(out_dir=None):
    """Definition 6.7 and Theorem 6.10: consistency, and an estimator that is not consistent."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    nmax, eps = 5000, 0.05
    grid = cons_grid(nmax)
    ax = axes[0, 0]
    P = cons_paths("bern", 20, nmax, 2024, grid)
    for j in range(20):
        ax.plot(grid, P[j], color=BLUE, lw=0.6, alpha=0.6)
    ax.axhspan(0.3 - eps, 0.3 + eps, color=GREEN, alpha=0.15, lw=0)
    ax.axhline(0.3, color="k", lw=0.8)
    ax.set_xscale("log")
    ax.set_ylim(0, 0.8)
    ax.set_xlabel("n")
    ax.set_ylabel(r"$\hat p_n$")
    ax.set_title(r"20 paths of $\hat p_n$, $p=0.3$")
    common.panel_label(ax, "a", loc="upper right")

    ax = axes[0, 1]
    ns = np.unique(np.round(np.logspace(0, np.log10(nmax), 120)).astype(int))
    ex = [tail_prob("bern", m, eps) for m in ns]
    ch = [min(1, cons_mse("bern", m) / eps ** 2) for m in ns]
    ho = [min(1, 2 * math.exp(-2 * m * eps ** 2)) for m in ns]
    ax.plot(ns, ex, color=BLUE, label="exact")
    ax.plot(ns, ch, color=ORANGE, ls="--", label=r"Chebyshev MSE$/\epsilon^2$")
    ax.plot(ns, ho, color=RED, ls="-.", label=r"Hoeffding $2e^{-2n\epsilon^2}$")
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_ylim(1e-6, 2)
    ax.set_xlabel("n")
    ax.set_title(r"$P(|\hat p_n-p|>0.05)$")
    ax.legend(fontsize=7.5, frameon=False, loc="lower left")
    common.panel_label(ax, "b", loc="upper right")

    ax = axes[0, 2]
    Pm = cons_paths("cmean", 8, nmax, 77, grid)
    Pd = cons_paths("cmed", 8, nmax, 77, grid)
    for j in range(8):
        ax.plot(grid, Pm[j], color=RED, lw=0.7, alpha=0.75, label="mean" if j == 0 else None)
        ax.plot(grid, Pd[j], color=BLUE, lw=0.7, alpha=0.85, label="median" if j == 0 else None)
    ax.axhline(0, color="k", lw=0.8)
    ax.set_xscale("log")
    ax.set_ylim(-6, 6)
    ax.set_xlabel("n")
    ax.set_title("Cauchy data: mean and median")
    ax.legend(fontsize=7.5, frameon=False, loc="lower right")
    common.panel_label(ax, "c", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig6-S3", out_dir)


def fig_6_s4(out_dir=None):
    """Examples 6.15 and 6.17, Section 6.5: exact coverage of three 95 percent intervals for p."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    cols = {"wald": RED, "wilson": BLUE, "hoeffding": GREEN}
    labs = {"wald": "Wald (6.10)", "wilson": "Wilson", "hoeffding": "Hoeffding (Ex. 6.15)"}
    ps = np.linspace(0.0005, 0.9995, 1999)
    ax = axes[0, 0]
    for m in ("hoeffding", "wilson", "wald"):
        ax.plot(ps, coverage_curve(50, ps, m), color=cols[m], lw=0.9, label=labs[m])
    ax.axhline(0.95, color="k", ls=":", lw=1)
    ax.set_ylim(0.75, 1.005)
    ax.set_xlabel("p")
    ax.set_ylabel("exact coverage")
    ax.set_title(r"$n=50$, coverage against $p$")
    ax.legend(fontsize=7.5, frameon=False, loc="lower center")
    common.panel_label(ax, "a", loc="upper left")

    ax = axes[0, 1]
    ns = np.arange(10, 201)
    for m in ("wilson", "wald"):
        ax.plot(ns, [binom_coverage(int(k), 0.2, m)[0] for k in ns], color=cols[m], lw=0.9, label=labs[m])
    ax.axhline(0.95, color="k", ls=":", lw=1)
    ax.set_ylim(0.85, 1.0)
    ax.set_xlabel("n")
    ax.set_title(r"$p=0.2$, coverage against $n$")
    ax.legend(fontsize=7.5, frameon=False, loc="lower right")
    common.panel_label(ax, "b", loc="upper left")

    ax = axes[0, 2]
    for m in ("hoeffding", "wilson", "wald"):
        ax.plot(ps, [binom_coverage(50, p, m)[1] for p in ps], color=cols[m], lw=1.2, label=labs[m])
    ax.set_xlabel("p")
    ax.set_ylabel("expected width")
    ax.set_title(r"$n=50$, expected width")
    ax.set_ylim(0, 0.42)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig6-S4", out_dir)


def days_study(D=2000, seed=6):
    """'Day j': a new, unrelated parameter and a 95 percent interval. Known-sigma z-intervals for a
    Normal mean (exact coverage) and Wald intervals for a poll with n = 30 and a random p_j."""
    rng = np.random.default_rng(seed)
    z = zq(0.05)
    mu = rng.uniform(-100, 100, D)
    nj = rng.integers(5, 60, D)
    xbar = mu + rng.standard_normal(D) / np.sqrt(nj)
    hit_z = np.abs(xbar - mu) < z / np.sqrt(nj)
    p = rng.uniform(0.02, 0.98, D)
    S = rng.binomial(30, p)
    lo, hi = binom_interval(S, 30, "wald")
    hit_w = (lo < p) & (p < hi)
    mean_cov_w = float(np.mean([binom_coverage(30, pj, "wald")[0] for pj in p]))
    days = np.arange(1, D + 1)
    return days, np.cumsum(hit_z) / days, np.cumsum(hit_w) / days, mean_cov_w


def fig_6_s5(out_dir=None):
    """Section 6.3.2: three readings of '95 percent'."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    ax = axes[0, 0]
    days, fz, fw, mcw = days_study()
    ax.plot(days, fz, color=BLUE, label="z-interval, exact 95%")
    ax.plot(days, fw, color=RED, label="Wald poll, n = 30")
    ax.axhline(0.95, color="k", ls=":", lw=1)
    ax.axhline(mcw, color=RED, ls="--", lw=0.9)
    ax.set_xscale("log")
    ax.set_ylim(0.7, 1.01)
    ax.set_xlabel("day")
    ax.set_ylabel("fraction of intervals that trap")
    ax.set_title("Unrelated parameters, one per day")
    ax.legend(fontsize=7.5, frameon=False, loc="lower right")
    common.panel_label(ax, "a", loc="upper right")

    ax = axes[0, 1]
    bw = berger_wolpert(16.0)
    rng = np.random.default_rng(614)
    X = rng.choice([-1, 1], size=(20000, 2))
    Y = 16.0 + X
    eq = Y[:, 0] == Y[:, 1]
    C = np.where(eq, Y[:, 0] - 1, Y.mean(axis=1))
    hit = C == 16.0
    vals = [hit.mean(), hit[eq].mean(), hit[~eq].mean()]
    ax.bar([0, 1, 2], vals, color=[GREY, ORANGE, BLUE], width=0.6)
    ax.plot([0, 1, 2], [bw["overall"], bw["given_equal"], bw["given_unequal"]], "k_", ms=28, mew=1.5)
    for i, v in enumerate(vals):
        ax.text(i, v + 0.02, "%.3f" % v, ha="center", fontsize=8.5)
    ax.set_xticks([0, 1, 2])
    ax.set_xticklabels(["all", r"$Y_1=Y_2$", r"$Y_1\neq Y_2$"])
    ax.set_ylim(0, 1.12)
    ax.set_ylabel("coverage")
    ax.set_title("Example 6.14 (20 000 runs)")

    common.panel_label(ax, "b", loc="upper left")
    ax = axes[0, 2]
    th = np.linspace(-4, 4, 801)
    for n, col in ((1, RED), (4, ORANGE), (25, BLUE)):
        ax.plot(th, bayes_coverage(th, n), color=col, label="n = %d" % n)
    ax.axhline(0.95, color="k", ls=":", lw=1)
    ax.set_ylim(0, 1.02)
    ax.set_xlabel(r"true mean $\theta$")
    ax.set_title(r"Posterior 95% interval, prior $N(0,1)$")
    ax.legend(fontsize=7.5, frameon=False, loc="lower center")
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig6-S5", out_dir)


def fig_6_s6(out_dir=None):
    """Exercises 6.2 and 6.3: four estimators of the endpoint of a Uniform(0, theta)."""
    fig, axes = common.panel_grid(1, 2, size=(3.9, 3.0))
    ax = axes[0, 0]
    ns = np.unique(np.round(np.logspace(0, 3, 60)).astype(int))
    cols = {"2 mean": BLUE, "max": RED, "(n+1)/n max": GREEN, "(n+2)/(n+1) max": ORANGE}
    labs = {"2 mean": r"$2\bar X_n$ (Ex. 6.3)", "max": r"$X_{(n)}$ (Ex. 6.2)",
            "(n+1)/n max": r"$\frac{n+1}{n}X_{(n)}$", "(n+2)/(n+1) max": r"$\frac{n+2}{n+1}X_{(n)}$"}
    for k in cols:
        ax.loglog(ns, [uniform_estimators(int(m))[k]["mse"] for m in ns], color=cols[k], label=labs[k])
    rng = np.random.default_rng(63)
    for m in (2, 5, 20, 100, 500):
        X = rng.uniform(0, 1, (20000, m))
        ax.plot(m, np.mean((2 * X.mean(1) - 1) ** 2), "o", ms=4, mfc="white", mec=BLUE)
        ax.plot(m, np.mean((X.max(1) - 1) ** 2), "o", ms=4, mfc="white", mec=RED)
    ax.set_xlabel("n")
    ax.set_ylabel(r"MSE / $\theta^2$")
    ax.set_title("MSE: order $1/n$ against $1/n^2$")
    ax.legend(fontsize=7.5, frameon=False, loc="lower left")
    common.panel_label(ax, "a", loc="upper right")

    ax = axes[0, 1]
    n = 10
    X = rng.uniform(0, 1, (40000, n))
    bins = np.linspace(0.3, 1.7, 71)
    ax.hist(2 * X.mean(1), bins=bins, density=True, color=BLUE, alpha=0.5, label=r"$2\bar X_n$")
    ax.hist(X.max(1), bins=bins, density=True, color=RED, alpha=0.5, label=r"$X_{(n)}$")
    t = np.linspace(0, 1, 400)
    ax.plot(t, n * t ** (n - 1), color=RED, lw=1)
    ax.axvline(1.0, color="k", lw=0.9)
    ax.set_xlabel(r"estimate ($\theta=1$, $n=10$)")
    ax.set_ylabel("density")
    ax.set_title("Sampling distributions")
    ax.legend(fontsize=7.5, frameon=False, loc="upper left")
    common.panel_label(ax, "b", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig6-S6", out_dir)


FIGURES = {"6.S1": fig_6_s1, "6.S2": fig_6_s2, "6.S3": fig_6_s3, "6.S4": fig_6_s4, "6.S5": fig_6_s5,
           "6.S6": fig_6_s6}

if __name__ == "__main__":
    common.run(FIGURES, CH)
