"""Murphy (2022), Chapter 4: Statistics.

Companion code for Chapter 4 of K. P. Murphy, Probabilistic Machine Learning: An Introduction
(MIT Press, 2022; April 2025 draft), pp. 107-166, written from scratch with numpy, scipy and
matplotlib, to be read next to the lecture notes (../ch04-statistics.html). scipy and scikit-learn
are only used to confirm the implementations.

Methods
* Section 4.2, maximum likelihood: mle_gauss (4.35)-(4.36), mle_mvn (4.43)-(4.51), ols (4.59).
* Section 4.3, surrogate losses (4.61)-(4.67): loss_01, loss_log2, loss_hinge, loss_exp.
* Section 4.4, method of moments for the uniform distribution (4.74) and the MLE (4.77);
  the recursive mean (4.81) and the exponentially weighted moving average with bias correction
  (4.82)-(4.88): ema.
* Section 4.5, regularisation: map_bernoulli (4.95), shrink_cov (4.97)-(4.98) and the Ledoit-Wolf
  closed-form intensity (ledoit_wolf), ridge_fit (4.99), ridge_cv (K-fold cross-validation with the
  one-standard-error rule, 4.101)-(4.106)), ridge_loo (leave-one-out through the hat matrix),
  learning curves (4.9).
* Section 4.6, Bayesian statistics: the beta-Bernoulli model (4.109)-(4.143), mixtures of betas
  (4.144)-(4.149), the Dirichlet-multinomial model (4.150)-(4.167), the Gaussian-Gaussian model
  (4.168)-(4.189), credible intervals and HPD regions (4.192)-(4.194), logistic regression with a
  Metropolis sampler (4.198)-(4.206), the grid and Laplace approximations (4.210)-(4.215).
* Section 4.7, frequentist statistics: the Fisher information and the Wald interval, the bootstrap
  (4.223), the Wald interval and its coverage (Section 4.7.5), bias and variance of the Gaussian
  variance estimators (4.234)-(4.236), the shrinkage estimator of a Gaussian mean (4.244)-(4.246),
  the bias-variance trade-off for ridge regression with Gaussian bumps (Figure 4.25).

Data. The iris measurements come from scikit-learn (Fisher, 1936). The polynomial examples (Figures
4.5, 4.7 and 4.9) use a simulated data set (a smooth truth plus Gaussian noise of standard deviation
2, seed fixed in the code): the book's own 21 points are not available, so the curves are close
stand-ins, not copies. Figure 4.8 (early stopping) uses the 8 x 8 digits from scikit-learn and a small
network trained with numpy instead of the IMDB text classifier of the book.

Figures. Reproduced here: 4.1-4.10, 4.12-4.20, 4.22-4.25 (4.6 is a drawn schematic built from the fold
indices, 4.21 is redrawn with computed numbers). Skipped, with the reason:
  4.11 sequential Bayesian updating tree (a diagram from Ortega et al., nothing is computed beyond
       Laplace's rule (4.131), which is in the notes);
  4.26 cartoon of the bias-variance trade-off against model complexity (a hand-drawn sketch).
Supplementary figures for the gaps filled in the notes:
  4.S1 the exact coverage of the Wald, Wilson and Jeffreys intervals for a Bernoulli parameter;
  4.S2 cross-validation estimates against the test error (the leave-one-out shortcut);
  4.S3 the method of moments against the MLE for the uniform distribution.
Run `python3 ch04_statistics.py --list`, `python3 ch04_statistics.py` (all figures, a minute or two)
or `python3 ch04_statistics.py --fig 4.5`. Files go to ../img/ch04/; the data used by the browser
demonstrations go to ../data/ch04-*.json.
"""
import json
import os
import sys

os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")

import numpy as np
from scipy import special, stats

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import common  # noqa: E402
from common import BLUE, CYAN, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, plt, savefig  # noqa: E402

CH = 4
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
SEED_POLY = 21


# =========================================================================== Section 4.2: MLE
def mle_gauss(y):
    """(4.35)-(4.36): sample mean and the 1/N variance."""
    y = np.asarray(y, float)
    mu = y.mean()
    return mu, ((y - mu) ** 2).mean()


def mle_mvn(Y):
    """(4.43), (4.51): mean and the 1/N covariance matrix of the rows of Y."""
    Y = np.asarray(Y, float)
    mu = Y.mean(0)
    C = Y - mu
    return mu, C.T @ C / len(Y)


def corr_from_cov(S):
    """(4.52)."""
    d = 1.0 / np.sqrt(np.diag(S))
    return d[:, None] * S * d[None, :]


def ols(X, y):
    """(4.59), by least squares (not by forming the inverse)."""
    return np.linalg.lstsq(X, y, rcond=None)[0]


# =========================================================================== Section 4.3: losses
def loss_01(z):
    return (np.asarray(z) < 0).astype(float)


def loss_log2(z):
    """The log loss (4.66) in bits, log2(1 + e^-z): the form that upper-bounds the 0-1 loss."""
    return np.logaddexp(0.0, -np.asarray(z)) / np.log(2.0)


def loss_hinge(z):
    return np.maximum(0.0, 1.0 - np.asarray(z))


def loss_exp(z):
    return np.exp(-np.asarray(z))


# =========================================================================== Section 4.4
def mom_uniform(y):
    """(4.74): method-of-moments estimate of (theta1, theta2) for Unif(theta1, theta2)."""
    y = np.asarray(y, float)
    m1, m2 = y.mean(), (y ** 2).mean()
    th1 = m1 - np.sqrt(3 * (m2 - m1 ** 2))
    return th1, 2 * m1 - th1


def mle_uniform(y):
    """(4.77): the sample minimum and maximum."""
    return float(np.min(y)), float(np.max(y))


def ema(y, beta, correct=False, mu0=0.0):
    """(4.82), started at mu0; correct=True divides by 1 - beta^t as in (4.88)."""
    out = np.empty(len(y))
    m = mu0
    for t, v in enumerate(y, start=1):
        m = beta * m + (1 - beta) * v
        out[t - 1] = m / (1 - beta ** t) if correct else m
    return out


# =========================================================================== Section 4.5
def map_bernoulli(n1, n0, a, b):
    """(4.95)."""
    return (n1 + a - 1) / (n1 + n0 + a + b - 2)


def shrink_cov(S_mle, lam):
    """(4.98): keep the diagonal, multiply the off-diagonal entries by (1 - lam)."""
    out = (1 - lam) * S_mle
    np.fill_diagonal(out, np.diag(S_mle))
    return out


def ledoit_wolf(X):
    """The closed-form shrinkage towards (tr S / D) I of Ledoit and Wolf (2004), which minimises the
    expected squared Frobenius loss over the family (1 - delta) S + delta mu I.
    Returns (estimate, delta). X is N x D."""
    X = np.asarray(X, float)
    N, D = X.shape
    Xc = X - X.mean(0)
    S = Xc.T @ Xc / N
    mu = np.trace(S) / D
    d2 = ((S - mu * np.eye(D)) ** 2).sum() / D
    b2 = 0.0
    for x in Xc:
        b2 += ((np.outer(x, x) - S) ** 2).sum() / D
    b2 = min(b2 / N ** 2, d2)
    delta = b2 / d2
    return (1 - delta) * S + delta * mu * np.eye(D), delta


def cond(S):
    w = np.linalg.eigvalsh(S)
    return w[-1] / max(w[0], 1e-300)


def poly_features(x, deg, centre=10.0, scale=10.0):
    """Legendre polynomials P_1(u), ..., P_deg(u) of u = (x - centre) / scale (without the constant):
    a degree-deg polynomial model in a well-conditioned basis, so that lambda = 0 is an honest
    least-squares fit."""
    u = (np.asarray(x, float) - centre) / scale
    return np.polynomial.legendre.legvander(u, deg)[:, 1:]


def ridge_fit(Phi, y, lam):
    """(4.99) with an unpenalised intercept: centre, then solve the augmented least-squares problem
    [Phi; sqrt(lam) I] w = [y; 0]. Returns (intercept, w). lam = 0 gives the minimum-norm solution."""
    mp, my = Phi.mean(0), y.mean()
    Pc = Phi - mp
    M = Phi.shape[1]
    if lam > 0:
        A = np.vstack([Pc, np.sqrt(lam) * np.eye(M)])
        b = np.concatenate([y - my, np.zeros(M)])
    else:
        A, b = Pc, y - my
    w = np.linalg.lstsq(A, b, rcond=1e-13)[0]
    return my - mp @ w, w


def ridge_predict(model, Phi):
    return model[0] + Phi @ model[1]


def kfold_indices(n, K, rng=None):
    """Folds as in Figure 4.6; a random permutation if rng is given, else contiguous blocks."""
    idx = np.arange(n) if rng is None else rng.permutation(n)
    return np.array_split(idx, K)


def ridge_cv(Phi, y, lams, K=5, rng=None):
    """(4.106) and Section 4.5.5.1: out-of-fold squared errors, the mean MSE and its standard error
    for each lambda. Returns mean, se and the per-example loss matrix (len(lams) x N)."""
    n = len(y)
    folds = kfold_indices(n, K, rng)
    L = np.zeros((len(lams), n))
    for k, te in enumerate(folds):
        tr = np.setdiff1d(np.arange(n), te)
        for i, lam in enumerate(lams):
            m = ridge_fit(Phi[tr], y[tr], lam)
            L[i, te] = (y[te] - ridge_predict(m, Phi[te])) ** 2
    return L.mean(1), L.std(1) / np.sqrt(n), L


def one_se_rule(lams, mean, se):
    """The simplest model (largest lambda) whose CV risk is within one se of the best."""
    ib = int(np.argmin(mean))
    ok = np.where(mean <= mean[ib] + se[ib])[0]
    return int(ok.max()), ib


def ridge_loo(Phi, y, lam):
    """Leave-one-out squared errors in closed form: e_i / (1 - H_ii), with H the hat matrix of the
    ridge fit with an unpenalised intercept (column of ones, unpenalised)."""
    n, M = Phi.shape
    X = np.hstack([np.ones((n, 1)), Phi])
    P = np.diag(np.r_[0.0, np.full(M, lam)])
    H = X @ np.linalg.solve(X.T @ X + P, X.T)
    e = y - H @ y
    return (e / (1 - np.diag(H))) ** 2


# ---- the simulated polynomial data (Figures 4.5, 4.7, 4.9)
def poly_truth(x):
    x = np.asarray(x, float)
    return 4.0 * np.sin(x / 3.0) + 0.25 * (x - 10.0)


def poly_data(n=21, sd=2.0, seed=SEED_POLY, ntest=500):
    rng = np.random.default_rng(seed)
    x = np.linspace(0, 20, n)
    y = poly_truth(x) + sd * rng.standard_normal(n)
    xt = rng.uniform(0, 20, ntest)
    yt = poly_truth(xt) + sd * rng.standard_normal(ntest)
    return x, y, xt, yt


# =========================================================================== Section 4.6
def beta_post(a, b, n1, n0):
    return a + n1, b + n0


def beta_binom_pmf(y, M, a, b):
    """(4.135)."""
    y = np.asarray(y)
    return np.exp(special.gammaln(M + 1) - special.gammaln(y + 1) - special.gammaln(M - y + 1)
                  + special.betaln(y + a, M - y + b) - special.betaln(a, b))


def log_marglik_beta_bern(n1, n0, a, b):
    """(4.143) without the binomial coefficient: log B(a + N1, b + N0) - log B(a, b)."""
    return special.betaln(a + n1, b + n0) - special.betaln(a, b)


def beta_mix_posterior(w, abs_, n1, n0):
    """(4.146)-(4.147): posterior weights and parameters of a mixture of betas."""
    logw = np.log(w) + np.array([log_marglik_beta_bern(n1, n0, a, b) for a, b in abs_])
    logw -= special.logsumexp(logw)
    return np.exp(logw), [(a + n1, b + n0) for a, b in abs_]


def beta_hpd(a, b, mass=0.95, n=200001):
    """The highest-density interval of a unimodal Beta(a, b) (a, b > 1) by the narrowest-interval
    search of Section 4.6.6: scan the lower tail probability p and minimise the width."""
    p = np.linspace(0, 1 - mass, n)
    lo = stats.beta.ppf(p, a, b)
    hi = stats.beta.ppf(p + mass, a, b)
    i = np.argmin(hi - lo)
    return lo[i], hi[i]


def dirichlet_logpdf(th, alpha):
    alpha = np.asarray(alpha, float)
    return (special.gammaln(alpha.sum()) - special.gammaln(alpha).sum()
            + ((alpha - 1) * np.log(th)).sum(-1))


def log_marglik_dirichlet(counts, alpha):
    """(4.165)-(4.167)."""
    counts, alpha = np.asarray(counts, float), np.asarray(alpha, float)
    return (special.gammaln(alpha.sum()) - special.gammaln(counts.sum() + alpha.sum())
            + (special.gammaln(counts + alpha) - special.gammaln(alpha)).sum())


def gauss_post_1d(y, sigma2, m0, tau2):
    """(4.170)-(4.171): posterior mean and variance of a Gaussian mean with known variance."""
    y = np.asarray(y, float)
    N = len(y)
    prec = N / sigma2 + 1 / tau2
    m = (y.sum() / sigma2 + m0 / tau2) / prec
    return m, 1 / prec


def gauss_post_mvn(Y, Sigma, m0, V0):
    """(4.188)-(4.189)."""
    Y = np.atleast_2d(Y)
    N = len(Y)
    Si = np.linalg.inv(Sigma)
    V = np.linalg.inv(np.linalg.inv(V0) + N * Si)
    m = V @ (Si @ (N * Y.mean(0)) + np.linalg.solve(V0, m0))
    return m, V


def hpd_grid(x, dens, mass=0.95):
    """HPD region of a density on a grid: the superlevel set {dens >= p*} with mass `mass`
    (4.193)-(4.194). Returns the threshold and a boolean mask."""
    dx = x[1] - x[0]
    order = np.argsort(dens)[::-1]
    c = np.cumsum(dens[order]) * dx
    k = int(np.searchsorted(c, mass))
    thr = dens[order][min(k, len(x) - 1)]
    return thr, dens >= thr


def logistic_nll_grad_hess(theta, x, y):
    """theta = (b, w); p = sigmoid(b + w x). Returns NLL, gradient, Hessian."""
    X = np.stack([np.ones_like(x), x], 1)
    p = common.sigmoid(X @ theta)
    nll = -np.sum(y * np.log(p + 1e-300) + (1 - y) * np.log(1 - p + 1e-300))
    g = X.T @ (p - y)
    H = (X * (p * (1 - p))[:, None]).T @ X
    return nll, g, H


def logistic_mle(x, y, iters=50):
    th = np.zeros(2)
    for _ in range(iters):
        _, g, H = logistic_nll_grad_hess(th, x, y)
        step = np.linalg.solve(H, g)
        th = th - step
        if np.abs(step).max() < 1e-12:
            break
    return th


def logistic_mh(x, y, n=60000, burn=10000, prior_sd=10.0, step=None, seed=0):
    """Random-walk Metropolis for the posterior of (b, w) under independent N(0, prior_sd^2) priors,
    with a proposal shaped by the inverse Hessian at the mode. Returns the retained draws."""
    rng = np.random.default_rng(seed)
    th0 = logistic_mle(x, y)
    _, _, H = logistic_nll_grad_hess(th0, x, y)
    L = np.linalg.cholesky(np.linalg.inv(H)) * (2.4 / np.sqrt(2) if step is None else step)

    def logp(t):
        return -logistic_nll_grad_hess(t, x, y)[0] - 0.5 * (t ** 2).sum() / prior_sd ** 2

    th, lp, out, acc = th0.copy(), logp(th0), np.empty((n, 2)), 0
    for i in range(n):
        prop = th + L @ rng.standard_normal(2)
        lq = logp(prop)
        if np.log(rng.random()) < lq - lp:
            th, lp, acc = prop, lq, acc + 1
        out[i] = th
    return out[burn:], acc / n


def wald_interval(k, n, z=1.959963984540054):
    p = k / n
    h = z * np.sqrt(p * (1 - p) / n)
    return p - h, p + h


def wilson_interval(k, n, z=1.959963984540054):
    p = k / n
    c = (p + z * z / (2 * n)) / (1 + z * z / n)
    h = z / (1 + z * z / n) * np.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
    return c - h, c + h


def jeffreys_interval(k, n, alpha=0.05):
    """Central credible interval of the Beta(k + 1/2, n - k + 1/2) posterior (Jeffreys prior)."""
    return (stats.beta.ppf(alpha / 2, k + .5, n - k + .5), stats.beta.ppf(1 - alpha / 2, k + .5, n - k + .5))


def bernoulli_coverage(theta, n, interval):
    """Exact probability that the interval returned by `interval(k, n)` covers theta."""
    ks = np.arange(n + 1)
    pk = stats.binom.pmf(ks, n, theta)
    cov = 0.0
    for k in ks:
        lo, hi = interval(k, n)
        if lo <= theta <= hi:
            cov += pk[k]
    return cov


# =========================================================================== Section 4.7
def bernoulli_fisher(theta):
    return 1.0 / (theta * (1 - theta))


def bootstrap_mle_bernoulli(y, B, rng):
    """Nonparametric bootstrap of the sample mean (4.223 and Section 4.7.3)."""
    n = len(y)
    idx = rng.integers(0, n, (B, n))
    return np.asarray(y)[idx].mean(1)


def mse_shrink(theta_star, theta0, sigma2, N, kappa0):
    """(4.244)-(4.246): bias, variance and MSE of the posterior mean under N(theta0, sigma2 / kappa0)."""
    w = N / (N + kappa0)
    bias = (1 - w) * (theta0 - theta_star)
    var = w ** 2 * sigma2 / N
    return bias, var, bias ** 2 + var


def var_mle_sigma2(sigma2, N):
    """Exercise 4.9: exact variance, bias and MSEs of the two variance estimators."""
    var_mle = 2 * (N - 1) / N ** 2 * sigma2 ** 2
    mse_mle = var_mle + (sigma2 / N) ** 2
    mse_unb = 2 * sigma2 ** 2 / (N - 1)
    return var_mle, mse_mle, mse_unb


# =========================================================================== small neural net (Figure 4.8)
def digits_split(seed=0, ntrain=60, nval=500):
    from sklearn.datasets import load_digits
    d = load_digits()
    X = d.data / 16.0
    y = d.target
    rng = np.random.default_rng(seed)
    p = rng.permutation(len(y))
    tr, va = p[:ntrain], p[ntrain:ntrain + nval]
    return X[tr], y[tr], X[va], y[va]


def mlp_train(Xtr, ytr, Xva, yva, hidden=64, epochs=60, lr=0.05, batch=10, seed=0):
    """One hidden layer (tanh), softmax output, minibatch SGD. Records the loss and accuracy per epoch."""
    rng = np.random.default_rng(seed)
    D, K = Xtr.shape[1], 10
    W1 = rng.standard_normal((D, hidden)) * np.sqrt(1.0 / D)
    b1 = np.zeros(hidden)
    W2 = rng.standard_normal((hidden, K)) * np.sqrt(1.0 / hidden)
    b2 = np.zeros(K)

    def fwd(X):
        H = np.tanh(X @ W1 + b1)
        Z = H @ W2 + b2
        Z = Z - Z.max(1, keepdims=True)
        P = np.exp(Z)
        P /= P.sum(1, keepdims=True)
        return H, P

    def stats_(X, y):
        _, P = fwd(X)
        return -np.log(P[np.arange(len(y)), y] + 1e-12).mean(), (P.argmax(1) == y).mean()

    hist = []
    for ep in range(epochs + 1):
        hist.append(stats_(Xtr, ytr) + stats_(Xva, yva))
        if ep == epochs:
            break
        perm = rng.permutation(len(ytr))
        for s in range(0, len(perm), batch):
            i = perm[s:s + batch]
            H, P = fwd(Xtr[i])
            G = P.copy()
            G[np.arange(len(i)), ytr[i]] -= 1
            G /= len(i)
            dH = (G @ W2.T) * (1 - H ** 2)
            W2 -= lr * H.T @ G
            b2 -= lr * G.sum(0)
            W1 -= lr * Xtr[i].T @ dH
            b1 -= lr * dH.sum(0)
    return np.array(hist)   # columns: train loss, train acc, val loss, val acc


# =========================================================================== figures
def _iris():
    from sklearn.datasets import load_iris
    d = load_iris()
    return d.data, d.target, d.feature_names


def fig_4_1(out=None):
    """Iris covariance and correlation matrices (lower triangle)."""
    X, _, names = _iris()
    mu, S = mle_mvn(X)
    R = corr_from_cov(S)
    short = ["sepal length", "sepal width", "petal length", "petal width"]
    fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.7))
    for ax, M, title, lab, cm, lim in ((axes[0], S, "Covariance matrix", "a", "RdBu_r", (-3.1, 3.1)),
                                       (axes[1], R, "Correlation matrix", "b", "RdBu_r", (-1, 1))):
        masked = np.where(np.tril(np.ones_like(M)) > 0, M, np.nan)
        im = ax.imshow(masked, cmap=cm, vmin=lim[0], vmax=lim[1])
        for i in range(4):
            for j in range(i + 1):
                ax.text(j, i, "%.2f" % M[i, j], ha="center", va="center", fontsize=8.5,
                        color="white" if abs(M[i, j]) > 0.6 * lim[1] else "black")
        ax.set_xticks(range(4))
        ax.set_yticks(range(4))
        ax.set_xticklabels(short, rotation=40, ha="right", fontsize=8)
        ax.set_yticklabels(short, fontsize=8)
        ax.set_title(title)
        fig.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
    fig.tight_layout()
    savefig(fig, CH, "fig4-1", out)


def fig_4_2(out=None):
    """Surrogate losses against the margin."""
    z = np.linspace(-2, 2, 801)
    fig, ax = plt.subplots(figsize=(4.8, 3.3))
    ax.plot(z, loss_01(z), color="k", label="0-1 loss")
    ax.plot(z, loss_hinge(z), color=RED, ls="--", label="hinge loss")
    ax.plot(z, loss_log2(z), color=BLUE, ls=":", lw=2.2, label="log loss (base 2)")
    ax.plot(z, loss_exp(z), color=GREEN, ls="-.", label="exp loss")
    ax.set_ylim(0, 3)
    ax.set_xlabel(r"margin $z=\tilde y\,\eta$")
    ax.set_ylabel("loss")
    ax.legend(frameon=False)
    ax.grid(alpha=0.25)
    savefig(fig, CH, "fig4-2", out)


def ema_signal(T=160, seed=3):
    rng = np.random.default_rng(seed)
    t = np.arange(1, T + 1)
    return -4.0 + 1.5 * np.sin(t / 18.0) + 1.2 * rng.standard_normal(T)


def fig_4_3(out=None):
    """Exponentially weighted moving average with and without bias correction."""
    y = ema_signal()
    fig, axes = plt.subplots(1, 2, figsize=(8.2, 3.1), sharey=True)
    for ax, beta, lab in zip(axes, (0.9, 0.99), "ab"):
        ax.plot(y, color="0.6", lw=0.9, label="signal")
        ax.plot(ema(y, beta), color=RED, label="EMA")
        ax.plot(ema(y, beta, correct=True), color=BLUE, ls="--", label="EMA with bias correction")
        ax.set_title(r"$\beta=%.2g$" % beta)
        ax.set_xlabel("time step")
        common.panel_label(ax, lab, "lower right")
    axes[0].legend(frameon=False, fontsize=8, loc="upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig4-3", out)


def cov_shrink_experiment(N, D=50, lam=0.9, seed=0, cond_true=10.0):
    rng = np.random.default_rng(seed)
    ev = np.logspace(0, -np.log10(cond_true), D)
    Q, _ = np.linalg.qr(rng.standard_normal((D, D)))
    Sigma = (Q * ev) @ Q.T
    X = rng.multivariate_normal(np.zeros(D), Sigma, size=N)
    _, S = mle_mvn(X)
    return Sigma, S, shrink_cov(S, lam)


def fig_4_4(out=None):
    """Eigenvalue spectrum of true, MLE and shrinkage covariance estimates in D = 50."""
    fig, axes = plt.subplots(1, 3, figsize=(10.4, 3.1), sharey=True)
    for ax, N, lab in zip(axes, (100, 50, 25), "abc"):
        Sigma, S, Sm = cov_shrink_experiment(N)
        for M, st, c, nm in ((Sigma, "-", "k", "true"), (S, ":", BLUE, "MLE"), (Sm, "--", RED, "MAP")):
            ev = np.sort(np.linalg.eigvalsh(M))[::-1]
            ax.plot(ev, st, color=c, lw=1.8, label="%s, k=%.3g" % (nm, cond(M)))
        ax.set_title("N=%d, D=50" % N)
        ax.set_xlabel("index")
        ax.set_ylim(-0.05, 2.2)
        ax.legend(frameon=False, fontsize=8, loc="upper right")
        common.panel_label(ax, lab, "lower left")
    axes[0].set_ylabel("eigenvalue")
    fig.tight_layout()
    savefig(fig, CH, "fig4-4", out)


LAMS = np.logspace(-4, 3, 36)


def _poly_ridge_curves(deg=14):
    x, y, xt, yt = poly_data()
    P, Pt = poly_features(x, deg), poly_features(xt, deg)
    tr, te = [], []
    for lam in LAMS:
        m = ridge_fit(P, y, lam)
        tr.append(np.mean((y - ridge_predict(m, P)) ** 2))
        te.append(np.mean((yt - ridge_predict(m, Pt)) ** 2))
    return np.array(tr), np.array(te)


def _lamtex(lam):
    if lam == 0:
        return "0"
    e = int(np.floor(np.log10(lam)))
    m = lam / 10 ** e
    return ("%g" % lam) if -2 <= e <= 3 else r"%.2g\times10^{%d}" % (m, e)


def fig_4_5(out=None):
    """Ridge regression on a degree-14 polynomial and 21 points: three fits and the MSE curve."""
    deg = 14
    x, y, xt, yt = poly_data()
    P = poly_features(x, deg)
    xs = np.linspace(0, 20, 400)
    Ps = poly_features(xs, deg)
    fig, axes = plt.subplots(1, 4, figsize=(12.6, 2.9))
    for ax, lam, lab in zip(axes[:3], (0.0, 2.5, 100.0), "abc"):
        m = ridge_fit(P, y, lam)
        ax.plot(xs, ridge_predict(m, Ps), color=BLUE)
        ax.plot(x, y, "o", color=RED, ms=4)
        ax.set_ylim(-9, 12)
        ax.set_xlim(0, 20)
        ax.set_title(r"$\lambda=%s$" % _lamtex(lam))
        common.panel_label(ax, lab, "lower left")
    tr, te = _poly_ridge_curves(deg)
    ax = axes[3]
    ax.semilogx(LAMS, te, color=RED, label="test")
    ax.semilogx(LAMS, tr, color=BLUE, label="train")
    ax.set_xlabel(r"$\lambda$")
    ax.set_ylabel("MSE")
    ax.legend(frameon=False)
    common.panel_label(ax, "d", "upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig4-5", out)


def fig_4_6(out=None):
    """The five folds of K-fold cross-validation, drawn from the fold indices."""
    n, K = 20, 5
    folds = kfold_indices(n, K)
    fig, ax = plt.subplots(figsize=(6.4, 2.3))
    for k, te in enumerate(folds):
        for i in range(n):
            ax.add_patch(plt.Rectangle((i, K - 1 - k), 1, 0.8, fc=ORANGE if i in te else "#9ec5f4", ec="white"))
        ax.text(-0.4, K - 1 - k + 0.4, "fold %d" % (k + 1), ha="right", va="center", fontsize=9)
    ax.text(n / 2, -0.55, "blue: training data, orange: held-out fold", ha="center", fontsize=9)
    ax.set_xlim(-4, n + 0.2)
    ax.set_ylim(-0.9, K)
    ax.axis("off")
    savefig(fig, CH, "fig4-6", out)


def cv_curves(deg=14, K=5):
    x, y, xt, yt = poly_data()
    P = poly_features(x, deg)
    mean, se, L = ridge_cv(P, y, LAMS, K)
    return mean, se, L


def fig_4_7(out=None):
    """Train and test MSE against lambda, and the 5-fold CV estimate with the one-standard-error rule."""
    tr, te = _poly_ridge_curves()
    mean, se, _ = cv_curves()
    i1, ib = one_se_rule(LAMS, mean, se)
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 3.2))
    ax = axes[0]
    ax.semilogx(LAMS, tr, color=BLUE, label="train MSE")
    ax.semilogx(LAMS, te, color=RED, label="test MSE")
    ax.set_xlabel(r"$\lambda$")
    ax.set_ylabel("mean squared error")
    ax.legend(frameon=False)
    common.panel_label(ax, "a", "upper left")
    ax = axes[1]
    ax.errorbar(LAMS, mean, yerr=se, color="k", ecolor=GREY, capsize=2, lw=1.2, elinewidth=0.9)
    ax.set_xscale("log")
    ax.axvline(LAMS[i1], color=BLUE, ls="--", label="one-SE choice")
    ax.axvline(LAMS[ib], color=GREEN, ls=":", label="minimum")
    ax.set_xlabel(r"$\lambda$")
    ax.set_ylabel("5-fold CV estimate of test MSE")
    ax.set_ylim(0, min(40, mean.max() * 1.05))
    ax.legend(frameon=False)
    common.panel_label(ax, "b", "upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig4-7", out)


def fig_4_8(out=None):
    """Early stopping, with a small network on digits (a stand-in for the IMDB text classifier)."""
    Xtr, ytr, Xva, yva = digits_split()
    h = mlp_train(Xtr, ytr, Xva, yva, epochs=100, lr=0.3, hidden=128)
    ep = np.arange(len(h))
    best = int(np.argmin(h[:, 2]))
    fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.1))
    axes[0].plot(ep, h[:, 0], color=BLUE, label="training loss")
    axes[0].plot(ep, h[:, 2], color=RED, label="validation loss")
    axes[0].axvline(best, color="k", ls=":", lw=1)
    axes[0].set_ylabel("cross-entropy loss")
    axes[1].plot(ep, h[:, 1], color=BLUE, label="training accuracy")
    axes[1].plot(ep, h[:, 3], color=RED, label="validation accuracy")
    axes[1].set_ylabel("accuracy")
    for ax, lab in zip(axes, "ab"):
        ax.set_xlabel("epoch")
        ax.legend(frameon=False, fontsize=8)
        common.panel_label(ax, lab, "center right")
    fig.tight_layout()
    savefig(fig, CH, "fig4-8", out)


def poly_truth2(x):
    return 1.0 + 0.5 * x + 0.6 * x ** 2   # quadratic truth for the learning curves, x in [-3, 3]


def learning_curves(degs=(1, 2, 10, 20), ns=None, reps=60, sd=2.0, seed=0, ntest=2000):
    ns = ns if ns is not None else np.unique(np.r_[np.arange(5, 30, 2), np.arange(30, 201, 10)])
    rng = np.random.default_rng(seed)
    xt = rng.uniform(-1, 1, ntest)
    yt = 1.0 + 1.5 * xt + 3.0 * xt ** 2 + sd * rng.standard_normal(ntest)  # truth: degree 2
    res = {d: np.zeros((len(ns), 2)) for d in degs}
    for r in range(reps):
        for i, n in enumerate(ns):
            x = rng.uniform(-1, 1, n)
            y = 1.0 + 1.5 * x + 3.0 * x ** 2 + sd * rng.standard_normal(n)
            for d in degs:
                A = np.vander(x, d + 1, increasing=True)
                w = np.linalg.lstsq(A, y, rcond=None)[0]
                res[d][i, 0] += np.mean((y - A @ w) ** 2) / reps
                res[d][i, 1] += np.mean((yt - np.vander(xt, d + 1, increasing=True) @ w) ** 2) / reps
    return ns, res


def fig_4_9(out=None):
    """Learning curves: training and test MSE against the training-set size, four polynomial degrees."""
    ns, res = learning_curves()
    fig, axes = plt.subplots(1, 4, figsize=(12.6, 2.9), sharey=True)
    for ax, d, lab in zip(axes, (1, 2, 10, 20), "abcd"):
        ax.plot(ns, res[d][:, 1], color=RED, label="test")
        ax.plot(ns, res[d][:, 0], color=BLUE, label="train")
        ax.axhline(4.0, color="k", lw=1)
        ax.set_title("truth: degree 2, model: degree %d" % d, fontsize=9)
        ax.set_xlabel("size of training set")
        ax.set_ylim(0, 20)
        common.panel_label(ax, lab, "upper right")
    axes[0].set_ylabel("mse")
    axes[0].legend(frameon=False, loc="center right")
    fig.tight_layout()
    savefig(fig, CH, "fig4-9", out)


def fig_4_10(out=None):
    """Updating a beta prior with a Bernoulli likelihood, N1 = 4, N0 = 1."""
    th = np.linspace(0, 1, 501)
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 3.1))
    for ax, (a, b), lab in zip(axes, ((2, 2), (1, 1)), "ab"):
        ax.plot(th, stats.beta.pdf(th, a, b), color=RED, label="prior Beta(%d, %d)" % (a, b))
        ax.plot(th, stats.beta.pdf(th, a + 4, b + 1), color=BLUE, label="posterior Beta(%d, %d)" % (a + 4, b + 1))
        ax.set_ylabel("prior / posterior density")
        ax.set_xlabel(r"$\theta$")
        ax2 = ax.twinx()
        ax2.plot(th, th ** 4 * (1 - th), color="k", ls="--", label="likelihood")
        ax2.set_ylabel("likelihood", color="k")
        h1, l1 = ax.get_legend_handles_labels()
        h2, l2 = ax2.get_legend_handles_labels()
        ax.legend(h1 + h2, l1 + l2, frameon=False, fontsize=8, loc="upper left")
        ax.set_ylim(0, 3.2)
        common.panel_label(ax, lab, "upper right")
    fig.tight_layout()
    savefig(fig, CH, "fig4-10", out)


def fig_4_12(out=None):
    """Posterior predictive for 10 future tosses against the plug-in approximation."""
    n1, n0, M = 4, 1, 10
    a, b = 1 + n1, 1 + n0
    y = np.arange(M + 1)
    fig, axes = plt.subplots(1, 2, figsize=(8.4, 3.0), sharey=True)
    axes[0].bar(y, beta_binom_pmf(y, M, a, b), color="#6b8fd6", ec="k", lw=0.6)
    th_hat = map_bernoulli(n1, n0, 1, 1)
    axes[1].bar(y, stats.binom.pmf(y, M, th_hat), color="#6b8fd6", ec="k", lw=0.6)
    axes[0].set_title("posterior predictive")
    axes[1].set_title(r"plug-in with $\hat\theta=%.1f$" % th_hat)
    for ax, lab in zip(axes, "ab"):
        ax.set_xticks(y)
        ax.set_xlabel("number of heads in 10 further tosses")
        common.panel_label(ax, lab, "upper left")
    axes[0].set_ylabel("probability")
    fig.tight_layout()
    savefig(fig, CH, "fig4-12", out)


MIX_W = (0.5, 0.5)
MIX_AB = ((20, 20), (30, 10))


def fig_4_13(out=None):
    """A mixture of two beta priors and its posterior after 20 heads and 10 tails."""
    th = np.linspace(0, 1, 801)
    prior = sum(w * stats.beta.pdf(th, a, b) for w, (a, b) in zip(MIX_W, MIX_AB))
    wp, ab_p = beta_mix_posterior(MIX_W, MIX_AB, 20, 10)
    post = sum(w * stats.beta.pdf(th, a, b) for w, (a, b) in zip(wp, ab_p))
    fig, ax = plt.subplots(figsize=(5.0, 3.2))
    ax.plot(th, prior, color=RED, label="prior")
    ax.plot(th, post, color=BLUE, ls="--", label="posterior")
    ax.set_xlabel(r"$\theta$")
    ax.set_ylabel("density")
    ax.legend(frameon=False)
    savefig(fig, CH, "fig4-13", out)


def _tri_xy(P):
    """Barycentric coordinates (rows of P sum to 1) to the plane of an equilateral triangle."""
    P = np.atleast_2d(P)
    return P[:, 1] + 0.5 * P[:, 2], (np.sqrt(3) / 2) * P[:, 2]


def _tri_density(ax, alpha, title, n=260):
    g = np.linspace(0, 1, n)
    A, B = np.meshgrid(g, g)
    C = 1 - A - B
    ok = (C > 1e-9) & (A > 1e-9) & (B > 1e-9)
    P = np.stack([np.clip(A, 1e-9, 1), np.clip(B, 1e-9, 1), np.clip(C, 1e-9, 1)], -1)
    ld = dirichlet_logpdf(P, alpha)
    top = np.nanmax(np.where(ok, ld, -np.inf))
    d = np.where(ok, np.clip((ld - top) / np.log(10), -3, 0), np.nan)
    x, y = _tri_xy(np.stack([A, B, C], -1).reshape(-1, 3))
    ax.pcolormesh(x.reshape(n, n), y.reshape(n, n), d, cmap="viridis", shading="auto", rasterized=True)
    ax.plot([0, 1, 0.5, 0], [0, 0, np.sqrt(3) / 2, 0], "k", lw=0.8)
    ax.set_aspect("equal")
    ax.axis("off")
    ax.set_title(title, fontsize=9)


def fig_4_14(out=None):
    """The Dirichlet density on the simplex for K = 3 (colour: log10 of the density relative to its maximum)."""
    fig, axes = plt.subplots(1, 4, figsize=(12.0, 3.0))
    ax = axes[0]
    rng = np.random.default_rng(1)
    smp = rng.dirichlet([2, 2, 2], 400)
    x, y = _tri_xy(smp)
    ax.plot([0, 1, 0.5, 0], [0, 0, np.sqrt(3) / 2, 0], "k", lw=0.8)
    ax.scatter(x, y, s=5, c=smp[:, 0], cmap="viridis")
    for (px, py), s in zip(((0, 0), (1, 0), (0.5, np.sqrt(3) / 2)), ("1", "2", "3")):
        ax.text(px, py - 0.07 if py == 0 else py + 0.04, s, ha="center", fontsize=9)
    ax.set_aspect("equal")
    ax.axis("off")
    ax.set_title(r"simplex $S_3$: draws from Dir(2,2,2)", fontsize=9)
    _tri_density(axes[1], (20, 20, 20), r"$\alpha=(20,20,20)$")
    _tri_density(axes[2], (3, 3, 20), r"$\alpha=(3,3,20)$")
    _tri_density(axes[3], (0.1, 0.1, 0.1), r"$\alpha=(0.1,0.1,0.1)$")
    for ax, lab in zip(axes, "abcd"):
        ax.text(0.0, 0.95, "(%s)" % lab, transform=ax.transAxes, fontsize=10)
    fig.tight_layout()
    savefig(fig, CH, "fig4-14", out)


def fig_4_15(out=None):
    """Samples from a five-dimensional symmetric Dirichlet distribution."""
    rng = np.random.default_rng(2)
    fig, axes = plt.subplots(2, 5, figsize=(9.6, 3.9), sharey=True)
    for r, (al, lab) in enumerate(((0.1, "a"), (1.0, "b"))):
        S = rng.dirichlet(np.full(5, al), 5)
        for c in range(5):
            axes[r, c].bar(range(1, 6), S[c], color="#6b8fd6", ec="k", lw=0.5)
            axes[r, c].set_ylim(0, 1)
            axes[r, c].set_xticks(range(1, 6))
        axes[r, 0].set_ylabel(r"$\alpha=%.1f$" % al)
    fig.tight_layout()
    savefig(fig, CH, "fig4-15", out)


def fig_4_16(out=None):
    """Inferring a Gaussian mean with known variance from a single observation, y = 3."""
    x = np.linspace(-5, 5, 801)
    sigma2, y = 1.0, 3.0
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 3.1), sharey=True)
    for ax, tau2, lab in zip(axes, (1.0, 5.0), "ab"):
        m, v = gauss_post_1d([y], sigma2, 0.0, tau2)
        ax.plot(x, stats.norm.pdf(x, 0, np.sqrt(tau2)), color=RED, label="prior")
        ax.plot(x, stats.norm.pdf(x, y, np.sqrt(sigma2)), color="k", ls=":", label="likelihood")
        ax.plot(x, stats.norm.pdf(x, m, np.sqrt(v)), color=BLUE, ls="--", label="posterior")
        ax.set_title("prior variance %g" % tau2)
        ax.set_xlabel(r"$\mu$")
        ax.legend(frameon=False, fontsize=8, loc="upper left")
        common.panel_label(ax, lab, "upper right")
    axes[0].set_ylabel("density")
    fig.tight_layout()
    savefig(fig, CH, "fig4-16", out)


def fig_4_17(out=None):
    """Inferring the mean of a two-dimensional Gaussian: data, prior and posterior after 10 points."""
    rng = np.random.default_rng(5)
    mu_true = np.array([0.5, 0.5])
    Sigma = 0.1 * np.array([[2.0, 1.0], [1.0, 1.0]])
    Y = rng.multivariate_normal(mu_true, Sigma, 10)
    V0 = 0.1 * np.eye(2)
    m, V = gauss_post_mvn(Y, Sigma, np.zeros(2), V0)
    fig, axes = plt.subplots(1, 3, figsize=(9.6, 3.2))
    for ax in axes:
        ax.set_xlim(-1, 1.3)
        ax.set_ylim(-1, 1.3)
        ax.set_aspect("equal")
        ax.plot(*mu_true, "x", color="k", ms=7)
    axes[0].plot(Y[:, 0], Y[:, 1], "o", color=RED, ms=4)
    axes[0].set_title("data")
    for ax, (mm, VV), t in ((axes[1], (np.zeros(2), V0), "prior"), (axes[2], (m, V), "posterior after 10 points")):
        for ns in (1, 2):
            ax.plot(*common.ellipse_xy(mm, VV, ns), color=BLUE)
        ax.plot(*mm, "o", color=BLUE, ms=3)
        ax.set_title(t)
    for ax, lab in zip(axes, "abc"):
        common.panel_label(ax, lab, "lower right")
    fig.tight_layout()
    savefig(fig, CH, "fig4-17", out)


def fig_4_18(out=None):
    """Central interval and HPD region of a Beta(3, 9) posterior."""
    a, b = 3, 9
    th = np.linspace(0, 1, 1001)
    d = stats.beta.pdf(th, a, b)
    ci = (stats.beta.ppf(0.025, a, b), stats.beta.ppf(0.975, a, b))
    hp = beta_hpd(a, b)
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 3.0), sharey=True)
    for ax, (lo, hi), t, lab in ((axes[0], ci, "central interval", "a"), (axes[1], hp, "HPD interval", "b")):
        ax.plot(th, d, "k")
        m = (th >= lo) & (th <= hi)
        ax.fill_between(th[m], d[m], color="#9ec5f4")
        ax.set_title("%s (%.2f, %.2f)" % (t, lo, hi))
        ax.set_xlabel(r"$\theta$")
        common.panel_label(ax, lab, "upper right")
    if hp[1] - hp[0] < ci[1] - ci[0]:
        axes[1].axhline(stats.beta.pdf(hp[0], a, b), color=RED, ls=":", lw=1)
    axes[0].set_ylabel("posterior density")
    fig.tight_layout()
    savefig(fig, CH, "fig4-18", out)


def fig_4_19(out=None):
    """Central interval and HPD region of a bimodal posterior (a mixture of two normals)."""
    x = np.linspace(-4, 10, 4001)
    d = 0.5 * stats.norm.pdf(x, 0, 1) + 0.5 * stats.norm.pdf(x, 5, 1.4)
    dx = x[1] - x[0]
    cdf = np.cumsum(d) * dx
    lo, hi = x[np.searchsorted(cdf, 0.025)], x[np.searchsorted(cdf, 0.975)]
    thr, mask = hpd_grid(x, d)
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 3.0), sharey=True)
    axes[0].plot(x, d, "k")
    m = (x >= lo) & (x <= hi)
    axes[0].fill_between(x[m], d[m], color="#9ec5f4")
    axes[0].set_title("central interval (%.2f, %.2f)" % (lo, hi), fontsize=10)
    axes[1].plot(x, d, "k")
    axes[1].fill_between(x, d, where=mask, color="#9ec5f4")
    axes[1].axhline(thr, color=RED, ls=":", lw=1)
    axes[1].text(9.9, thr + 0.006, r"$p^*$", ha="right", color=RED)
    axes[1].set_title("HPD region (two pieces)", fontsize=10)
    for ax, lab in zip(axes, "ab"):
        ax.set_xlabel(r"$\theta$")
        common.panel_label(ax, lab, "upper right")
    axes[0].set_ylabel("posterior density")
    fig.tight_layout()
    savefig(fig, CH, "fig4-19", out)


def iris_setosa_versicolor():
    X, y, _ = _iris()
    m = y < 2
    return X[m, 0], y[m].astype(float)


def fig_4_20(out=None):
    """Logistic regression on iris sepal length (setosa vs versicolor): plug-in and Bayesian predictions."""
    x, y = iris_setosa_versicolor()
    xc = x - x.mean()
    th = logistic_mle(xc, y)
    draws, acc = logistic_mh(xc, y)
    xs = np.linspace(4.0, 7.6, 300)
    xsc = xs - x.mean()
    sig = common.sigmoid
    fig, axes = plt.subplots(1, 2, figsize=(9.2, 3.2), sharey=True)
    rng = np.random.default_rng(0)
    jit = 0.04 * rng.standard_normal(len(y))
    for ax in axes:
        ax.plot(x, y + jit, "o", color="k", ms=3, alpha=0.6)
        ax.set_xlabel("sepal length")
    axes[0].plot(xs, sig(th[0] + th[1] * xsc), color=BLUE)
    xstar = x.mean() - th[0] / th[1]
    axes[0].axvline(xstar, color="k", lw=1)
    axes[0].set_ylabel("p(y=1)")
    sub = draws[::40]
    P = sig(sub[:, [0]] + sub[:, [1]] * xsc[None, :])
    for p in P[:60]:
        axes[1].plot(xs, p, color=BLUE, alpha=0.1, lw=0.8)
    axes[1].plot(xs, P.mean(0), color=BLUE, lw=2)
    lo, hi = np.percentile(P, [2.5, 97.5], axis=0)
    axes[1].fill_between(xs, lo, hi, color=BLUE, alpha=0.15)
    db = x.mean() - draws[:, 0] / draws[:, 1]
    q = np.percentile(db, [2.5, 97.5])
    axes[1].plot(q, [-0.04, -0.04], color="k", lw=5, solid_capstyle="butt")
    common.panel_label(axes[0], "a", "upper left")
    common.panel_label(axes[1], "b", "upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig4-20", out)


def fig_4_21(out=None):
    """Two delivery-time distributions and a deadline (redrawn from the description in the book)."""
    t = np.linspace(0, 30, 801)
    A, B, dl = (10.0, 3.5), (13.0, 1.0), 15.0
    fig, ax = plt.subplots(figsize=(5.4, 3.0))
    ax.plot(t, stats.norm.pdf(t, *A), color=RED, label="company A: mean %g, sd %g" % A)
    ax.plot(t, stats.norm.pdf(t, *B), color=BLUE, label="company B: mean %g, sd %g" % B)
    ax.axvline(dl, color="k", ls="--")
    ax.text(dl + 0.3, 0.37, "deadline", fontsize=9)
    ax.set_xlabel("delivery time")
    ax.set_ylabel("density")
    ax.set_title("P(late): A %.3f, B %.3f" % (stats.norm.sf(dl, *A), stats.norm.sf(dl, *B)), fontsize=10)
    ax.legend(frameon=False, fontsize=8, loc="upper left")
    savefig(fig, CH, "fig4-21", out)


def laplace_beta(a, b):
    """Laplace approximation of Beta(a, b) in theta: mode, and the Hessian of the energy there."""
    mode = (a - 1) / (a + b - 2)
    H = (a - 1) / mode ** 2 + (b - 1) / (1 - mode) ** 2
    return mode, 1.0 / H


def fig_4_22(out=None):
    """Grid and Laplace approximations to the posterior after 10 heads and 1 tail, uniform prior."""
    a, b = 11, 2
    th = np.linspace(0, 1, 801)
    exact = stats.beta.pdf(th, a, b)
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 3.1), sharey=True)
    grid = np.linspace(0, 1, 20)
    post = grid ** 10 * (1 - grid)
    post = post / post.sum() / (grid[1] - grid[0])
    axes[0].plot(th, exact, "k", label="true posterior")
    axes[0].plot(grid, post, "o-", color=RED, ms=4, lw=1, label="grid approximation")
    mode, v = laplace_beta(a, b)
    axes[1].plot(th, exact, "k", label="true posterior")
    axes[1].plot(th, stats.norm.pdf(th, mode, np.sqrt(v)), color=RED, ls="--", label="Laplace in " r"$\theta$")
    # Laplace in the logit space, mapped back (change of variables)
    # log density of eta = logit(theta): theta^a (1-theta)^b, mode a/(a+b), curvature (a+b) m (1-m)
    m = a / (a + b)
    veta = 1.0 / ((a + b) * m * (1 - m))
    eta = np.log(th[1:-1] / (1 - th[1:-1]))
    dens = stats.norm.pdf(eta, np.log(m / (1 - m)), np.sqrt(veta)) / (th[1:-1] * (1 - th[1:-1]))
    axes[1].plot(th[1:-1], dens, color=BLUE, ls=":", lw=2, label="Laplace in logit " r"$\theta$")
    for ax, lab in zip(axes, "ab"):
        ax.set_xlabel(r"$\theta$")
        ax.legend(frameon=False, fontsize=8, loc="upper left")
        common.panel_label(ax, lab, "upper right")
    axes[0].set_ylabel(r"$p(\theta|D)$")
    axes[1].set_ylim(0, 5.5)
    fig.tight_layout()
    savefig(fig, CH, "fig4-22", out)


def fig_4_23(out=None, B=10000, seed=7):
    """Bootstrap distribution of the Bernoulli MLE against the posterior, N = 10 and N = 100."""
    rng = np.random.default_rng(seed)
    fig, axes = plt.subplots(2, 2, figsize=(8.4, 5.2), sharex=True)
    for c, N in enumerate((10, 100)):
        y = (rng.random(N) < 0.7).astype(int)
        th = y.mean()
        bs = bootstrap_mle_bernoulli(y, B, rng)
        ps = rng.beta(1 + y.sum(), 1 + N - y.sum(), B)
        for r, (smp, nm) in enumerate(((bs, "Bootstrap"), (ps, "Bayes"))):
            ax = axes[r, c]
            ax.hist(smp, bins=np.linspace(0, 1, 41), color="#6b8fd6", ec="white", lw=0.4)
            ax.axvline(th, color="k", ls="--", lw=1.2)
            ax.set_title("%s: N = %d" % (nm, N), fontsize=10)
            common.panel_label(ax, "abcd"[2 * r + c], "upper left")
    axes[1, 0].set_xlabel(r"$\theta$")
    axes[1, 1].set_xlabel(r"$\theta$")
    fig.tight_layout()
    savefig(fig, CH, "fig4-23", out)


def fig_4_24(out=None):
    """Sampling distribution of the posterior mean for several prior strengths, and its relative MSE."""
    th_star, th0, sigma2, n = 1.0, 0.0, 1.0, 5
    x = np.linspace(-1, 2.5, 600)
    fig, axes = plt.subplots(1, 2, figsize=(9.2, 3.2))
    cols = (BLUE, RED, GREEN, MAGENTA)
    for k0, c in zip((0, 1, 2, 3), cols):
        bias, var, _ = mse_shrink(th_star, th0, sigma2, n, k0)
        axes[0].plot(x, stats.norm.pdf(x, th_star + bias, np.sqrt(var)), color=c, label=r"$\kappa_0=%d$" % k0)
    axes[0].set_xlabel("estimate")
    axes[0].set_title("sampling distribution, n = 5", fontsize=10)
    axes[0].legend(frameon=False, fontsize=8)
    Ns = np.arange(5, 50)
    for k0, c in zip((0, 1, 2, 3), cols):
        axes[1].plot(Ns, [mse_shrink(th_star, th0, sigma2, N, k0)[2] / mse_shrink(th_star, th0, sigma2, N, 0)[2] for N in Ns],
                     color=c, label=r"$\kappa_0=%d$" % k0)
    axes[1].set_xlabel("sample size")
    axes[1].set_ylabel("MSE of posterior mean / MSE of MLE")
    axes[1].axhline(1, color="k", lw=0.6)
    axes[1].legend(frameon=False, fontsize=8)
    for ax, lab in zip(axes, "ab"):
        common.panel_label(ax, lab, "upper right" if lab == "a" else "lower right")
    fig.tight_layout()
    savefig(fig, CH, "fig4-24", out)


def rbf_design(x, centres, width):
    return np.exp(-0.5 * ((x[:, None] - centres[None, :]) / width) ** 2)


def ridge_rbf_fits(lnlam, nsets=100, N=25, M=25, width=0.1, sd=0.3, seed=11):
    """Ridge fits (penalty (lam/2)||w||^2 on all weights, bias included, as in Bishop's Figure 3.5) to
    nsets noisy samples of sin(2 pi x); returns the prediction grid and the matrix of fits."""
    rng = np.random.default_rng(seed)
    c = np.linspace(0, 1, M)
    xs = np.linspace(0, 1, 200)
    Ps = rbf_design(xs, c, width)
    fits = np.zeros((nsets, len(xs)))
    lam = np.exp(lnlam)
    for s in range(nsets):
        x = rng.random(N)
        t = np.sin(2 * np.pi * x) + sd * rng.standard_normal(N)
        P = rbf_design(x, c, width)
        w = np.linalg.solve(P.T @ P + lam * np.eye(M), P.T @ t)
        fits[s] = Ps @ w
    return xs, fits


def fig_4_25(out=None):
    """Bias and variance of ridge regression with 25 Gaussian bumps: 20 fits and the average of 100."""
    fig, axes = plt.subplots(2, 2, figsize=(8.0, 5.4), sharex=True, sharey=True)
    for r, lnlam in enumerate((5.0, -5.0)):
        xs, fits = ridge_rbf_fits(lnlam)
        for f in fits[:20]:
            axes[r, 0].plot(xs, f, color=RED, lw=0.8, alpha=0.7)
        axes[r, 1].plot(xs, fits.mean(0), color=RED, label="average of 100 fits")
        axes[r, 1].plot(xs, np.sin(2 * np.pi * xs), color=GREEN, label=r"$\sin(2\pi x)$")
        axes[r, 0].set_ylabel(r"$\ln\lambda=%d$" % lnlam)
    axes[0, 1].legend(frameon=False, fontsize=8, loc="upper right")
    axes[0, 0].set_title("20 fits", fontsize=10)
    axes[0, 1].set_title("average over 100 data sets", fontsize=10)
    axes[0, 0].set_ylim(-1.6, 1.6)
    fig.tight_layout()
    savefig(fig, CH, "fig4-25", out)


# ----- supplementary figures
def fig_4_S1(out=None):
    """Exact coverage of three 95% intervals for a Bernoulli parameter (Section 4.7.5)."""
    th = np.linspace(0.005, 0.995, 400)
    fig, axes = plt.subplots(1, 2, figsize=(9.2, 3.2), sharey=True)
    for ax, n in zip(axes, (10, 50)):
        for fn, c, nm in ((wald_interval, RED, "Wald"), (wilson_interval, GREEN, "Wilson"),
                          (jeffreys_interval, BLUE, "Jeffreys credible")):
            def iv(k, n_, fn=fn):
                if fn is wald_interval and k in (0, n_):
                    return (k / n_, k / n_)
                return fn(k, n_)
            ax.plot(th, [bernoulli_coverage(t, n, iv) for t in th], color=c, lw=1, label=nm)
        ax.axhline(0.95, color="k", ls=":")
        ax.set_title("N = %d" % n)
        ax.set_xlabel(r"$\theta^*$")
        ax.set_ylim(0.5, 1.0)
    axes[0].set_ylabel("coverage probability")
    axes[0].legend(frameon=False, fontsize=8, loc="lower center")
    fig.tight_layout()
    savefig(fig, CH, "fig4-S1", out)


def fig_4_S2(out=None):
    """Five-fold CV, leave-one-out CV and the test error against lambda for the degree-14 ridge fit."""
    x, y, xt, yt = poly_data()
    P = poly_features(x, 14)
    mean5, se5, _ = ridge_cv(P, y, LAMS, 5)
    loo = np.array([ridge_loo(P, y, lam).mean() for lam in LAMS])
    _, te = _poly_ridge_curves()
    fig, ax = plt.subplots(figsize=(5.2, 3.2))
    ax.semilogx(LAMS, te, color=RED, label="test MSE")
    ax.semilogx(LAMS, mean5, color=BLUE, label="5-fold CV")
    ax.semilogx(LAMS, loo, color=GREEN, ls="--", label="leave-one-out (closed form)")
    ax.set_ylim(0, 40)
    ax.set_xlabel(r"$\lambda$")
    ax.set_ylabel("MSE")
    ax.legend(frameon=False, fontsize=8)
    savefig(fig, CH, "fig4-S2", out)


def fig_4_S3(out=None):
    """Method of moments and the MLE for Unif(0, 1) in repeated samples (Section 4.4.1.2)."""
    rng = np.random.default_rng(0)
    n, R = 5, 4000
    mom, mle = np.zeros((R, 2)), np.zeros((R, 2))
    for r in range(R):
        y = rng.random(n)
        mom[r], mle[r] = mom_uniform(y), mle_uniform(y)
    fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.1), sharey=True)
    for ax, (arr, nm) in zip(axes, ((mom, "method of moments"), (mle, "maximum likelihood"))):
        ax.hist(arr[:, 1], bins=np.linspace(0.3, 1.6, 53), color="#6b8fd6", ec="white", lw=0.3)
        ax.axvline(1.0, color="k", ls="--")
        ax.set_title(r"%s: $\hat\theta_2$, N = 5" % nm, fontsize=10)
    axes[0].set_ylabel("count of 4000 samples")
    fig.tight_layout()
    savefig(fig, CH, "fig4-S3", out)


FIGURES = {"4.1": fig_4_1, "4.2": fig_4_2, "4.3": fig_4_3, "4.4": fig_4_4, "4.5": fig_4_5, "4.6": fig_4_6,
           "4.7": fig_4_7, "4.8": fig_4_8, "4.9": fig_4_9, "4.10": fig_4_10, "4.12": fig_4_12, "4.13": fig_4_13,
           "4.14": fig_4_14, "4.15": fig_4_15, "4.16": fig_4_16, "4.17": fig_4_17, "4.18": fig_4_18,
           "4.19": fig_4_19, "4.20": fig_4_20, "4.21": fig_4_21, "4.22": fig_4_22, "4.23": fig_4_23,
           "4.24": fig_4_24, "4.25": fig_4_25, "4.S1": fig_4_S1, "4.S2": fig_4_S2, "4.S3": fig_4_S3}


# =========================================================================== demo data
def export_json():
    """Data for the browser demonstrations: the polynomial example of Figures 4.5 and 4.7, with the
    reference MSE and cross-validation curves the ridge demonstration must reproduce."""
    os.makedirs(DATA_OUT, exist_ok=True)
    x, y, xt, yt = poly_data()
    deg = 14
    P, Pt = poly_features(x, deg), poly_features(xt, deg)
    lam_grid = [float(v) for v in LAMS]
    tr, te = _poly_ridge_curves(deg)
    mean, se, _ = ridge_cv(P, y, LAMS, 5)
    i1, ib = one_se_rule(LAMS, mean, se)
    d = dict(x=[round(float(v), 6) for v in x], y=[round(float(v), 6) for v in y],
             xtest=[round(float(v), 5) for v in xt], ytest=[round(float(v), 5) for v in yt],
             degree=deg, centre=10.0, scale=10.0, truth="4 sin(x/3) + 0.25 (x - 10)", noise_sd=2.0,
             ref=dict(lams=lam_grid, train=[float(v) for v in tr], test=[float(v) for v in te],
                      cv=[float(v) for v in mean], cvse=[float(v) for v in se], one_se_index=i1, min_index=ib))
    path = os.path.join(DATA_OUT, "ch04-ridge.json")
    with open(path, "w") as f:
        json.dump(d, f, separators=(",", ":"))
    print("wrote", os.path.relpath(path, os.getcwd()))


if __name__ == "__main__":
    if "--json" in sys.argv:
        export_json()
    else:
        common.run(FIGURES, CH)
        if "--fig" not in sys.argv and "--list" not in sys.argv:
            export_json()
