"""Chapter 3 (Statistics) of Murphy (2023), Probabilistic Machine Learning: Advanced Topics.

Reproduces the computed figures of the chapter and of Supplement Chapter 3 (Bayesian statistics), and holds
the small numerical routines the lecture notes and the demonstrations rely on (conjugate updates, the
HPD interval, Fisher information, Jeffreys priors, the hierarchical models, information criteria, tests as
linear models, the number game and the healthy levels game).

Run   python3 ch03_statistics.py --list    or    python3 ch03_statistics.py --fig 3.4 3.15

Figures that are not reproduced, with the reason:
  3.9, 3.14, 3.19, 3.21, 3.30 (graphical models), 3.24 (schematic of K-fold cross validation): hand-drawn
      diagrams with nothing computed.
  3.22 (Bayesian Occam's razor): a schematic in the book; here a computed version for a coin is made.
  3.26 (waffle divorce posterior predictive): the 50-state data set is not bundled with any library on
      this machine and is not invented here; the method is the one of Figure 3.3 (see the notes).
  S3.1 (human responses in the numbers game): empirical data of a psychology experiment, not available.
  S3.7 (empirical distributions of durations and people's predictions): empirical data, not available.
Where the book's data are not bundled (the synthetic samples of Figures 3.3, 3.5, 3.8, 3.27, 3.28, 3.29 and
S3.5, S3.6), a seeded simulation with the stated parameters replaces them, and each caption says so.
The rat tumour counts and the eight-schools data are the published tables of Gelman et al. (2014);
Newcomb's measurements are the 66 values shipped with R's MASS package.
"""
import os
import sys

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import json
import numpy as np
from scipy import stats, special, optimize

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from common import (plt, savefig, run, panel_grid, panel_label, RED, BLUE, GREEN, MAGENTA, CYAN,  # noqa: E402
                    ORANGE, PURPLE, GREY, ellipse_xy)

CH = 3
JSON_DIR = os.path.normpath(os.path.join(HERE, "..", "data"))

# ============================================================================= data
SCHOOLS_Y = np.array([28.0, 8.0, -3.0, 7.0, -1.0, 1.0, 18.0, 12.0])
SCHOOLS_S = np.array([15.0, 10.0, 16.0, 11.0, 9.0, 11.0, 10.0, 18.0])
RATS_Y = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 5,
                   2, 5, 3, 2, 7, 7, 3, 3, 2, 9, 10, 4, 4, 4, 4, 4, 4, 4, 10, 4, 4, 4, 5, 11, 12, 5, 5, 6, 5, 6, 6, 6,
                   6, 16, 15, 15, 9, 4], dtype=float)
RATS_N = np.array([20, 20, 20, 20, 20, 20, 20, 19, 19, 19, 19, 18, 18, 17, 20, 20, 20, 20, 19, 19, 18, 18, 25, 24, 23,
                   20, 20, 20, 20, 20, 20, 10, 49, 19, 46, 27, 17, 49, 47, 20, 20, 13, 48, 50, 20, 20, 20, 20, 20, 20,
                   20, 48, 19, 19, 19, 22, 46, 49, 20, 20, 23, 19, 22, 20, 20, 20, 52, 46, 47, 24, 14], dtype=float)
NEWCOMB = np.array([28, -44, 29, 30, 24, 28, 37, 32, 36, 27, 26, 28, 29, 26, 27, 22, 23, 20, 25, 25, 36, 23, 31, 32,
                    24, 27, 33, 16, 24, 29, 36, 21, 28, 26, 27, 27, 32, 25, 28, 24, 40, 21, 31, 32, 28, 26, 30, 27,
                    26, 24, 32, 29, 34, -2, 25, 19, 36, 29, 30, 22, 28, 33, 39, 25, 16, 23], dtype=float)


def _out(out_dir):
    return out_dir


def _save(fig, name, out_dir):
    return savefig(fig, CH, name, out_dir=out_dir)


# ============================================================================= Section 3.2: Bayesian statistics
def beta_pdf(x, a, b):
    return stats.beta.pdf(x, a, b)


def beta_logml(n1, n0, a, b):
    """log p(D) for one particular sequence with n1 heads and n0 tails under a Beta(a, b) prior, (3.30)."""
    return special.betaln(a + n1, b + n0) - special.betaln(a, b)


def beta_hpd(a, b, mass=0.95):
    """Highest posterior density interval of Beta(a, b), a, b > 1, (3.19)-(3.20): the shortest interval with
    the given mass, found by sliding the lower tail probability."""
    tail = 1.0 - mass
    f = lambda p: stats.beta.ppf(p + mass, a, b) - stats.beta.ppf(p, a, b)
    r = optimize.minimize_scalar(f, bounds=(1e-12, tail - 1e-12), method="bounded", options={"xatol": 1e-12})
    return stats.beta.ppf(r.x, a, b), stats.beta.ppf(r.x + mass, a, b)


def mix_beta_posterior(w, ab, n1, n0):
    """Posterior of a mixture of betas, (3.208)-(3.210): new weights and parameters."""
    w = np.asarray(w, float)
    lw = np.log(w) + np.array([beta_logml(n1, n0, a, b) for a, b in ab])
    lw -= special.logsumexp(lw)
    return np.exp(lw), [(a + n1, b + n0) for a, b in ab]


def fig_3_1(out_dir=None):
    """Updating a Beta prior with a Bernoulli likelihood, N1 = 4, N0 = 1."""
    th = np.linspace(0, 1, 501)
    fig, axes = panel_grid(1, 2, size=(3.4, 2.7))
    for ax, (a, b), lab in zip(axes[0], [(1, 1), (2, 2)], "ab"):
        lik = th ** 4 * (1 - th)
        ax.plot(th, beta_pdf(th, a, b), "k", label="prior Beta(%d, %d)" % (a, b))
        ax.plot(th, beta_pdf(th, a + 4, b + 1), BLUE, label="posterior Beta(%d, %d)" % (a + 4, b + 1))
        ax2 = ax.twinx()
        ax2.plot(th, lik, RED, ls="--", label="likelihood")
        ax2.set_yticks([])
        ax.set_xlabel(r"$\theta$")
        ax.set_ylim(0, 2.7)
        h1, l1 = ax.get_legend_handles_labels()
        h2, l2 = ax2.get_legend_handles_labels()
        ax.legend(h1 + h2, l1 + l2, loc="upper left", fontsize=7.5, frameon=False)
        panel_label(ax, lab, "upper right")
    fig.tight_layout()
    return _save(fig, "fig3-1", out_dir)


def fig_3_2(out_dir=None):
    """Central interval and HPD region of a Beta(3, 9) posterior."""
    a, b = 3, 9
    th = np.linspace(0, 1, 801)
    lo, hi = stats.beta.ppf([0.025, 0.975], a, b)
    hl, hh = beta_hpd(a, b)
    fig, axes = panel_grid(1, 2, size=(3.4, 2.7))
    for ax, (l, h), lab, name in zip(axes[0], [(lo, hi), (hl, hh)], "ab", ["central interval", "HPD"]):
        ax.plot(th, beta_pdf(th, a, b), "k")
        m = (th >= l) & (th <= h)
        ax.fill_between(th[m], 0, beta_pdf(th[m], a, b), color=BLUE, alpha=0.3)
        ax.axvline(l, color=BLUE, lw=0.8)
        ax.axvline(h, color=BLUE, lw=0.8)
        ax.axhline(beta_pdf(l, a, b), color=GREY, lw=0.8, ls=":")
        ax.set_title("%s (%.2f, %.2f)" % (name, l, h), fontsize=9)
        ax.set_xlabel(r"$\theta$")
        panel_label(ax, lab, "upper right")
    fig.tight_layout()
    return _save(fig, "fig3-2", out_dir)


def poly_design(x, deg):
    return np.vander(np.asarray(x, float), deg + 1, increasing=True)


def linreg_plugin_and_bayes(x, y, xs, deg):
    """Plug-in (MLE) predictive mean and sd, and the Bayesian posterior predictive under the reference prior
    p(w, sigma^2) proportional to 1/sigma^2: a Student t with N - D degrees of freedom."""
    X, Xs = poly_design(x, deg), poly_design(xs, deg)
    N, D = X.shape
    G = np.linalg.inv(X.T @ X)
    w = G @ X.T @ y
    rss = float(np.sum((y - X @ w) ** 2))
    mle_sd = np.sqrt(rss / N) * np.ones(len(xs))
    s2 = rss / (N - D)
    nu = N - D
    scale = np.sqrt(s2 * (1 + np.einsum("ij,jk,ik->i", Xs, G, Xs)))
    sd = scale * np.sqrt(nu / (nu - 2.0)) if nu > 2 else np.full(len(xs), np.nan)
    return Xs @ w, mle_sd, sd, scale, nu


def fig_3_3(out_dir=None):
    """Plug-in versus Bayesian posterior predictive for a quadratic regression on a small data set (simulated)."""
    rng = np.random.default_rng(3)
    x = np.sort(rng.uniform(-5, 1, 8))
    y = 8 + 1.0 * x + 1.3 * x ** 2 + rng.normal(0, 4.0, x.size)
    xs = np.linspace(-6, 6, 241)
    mu, sd_mle, sd_b, _, nu = linreg_plugin_and_bayes(x, y, xs, 2)
    fig, axes = panel_grid(1, 2, size=(3.5, 2.8), sharey=True)
    for ax, sd, lab, ttl in zip(axes[0], [sd_mle, sd_b], "ab", ["plug-in (MLE)", "Bayesian predictive"]):
        ax.plot(x, y, "ko", ms=4, label="training data")
        ax.plot(xs, mu, RED, label="prediction")
        ax.fill_between(xs, mu - sd, mu + sd, color=RED, alpha=0.2)
        ax.set_xlabel("x")
        ax.set_title(ttl, fontsize=9)
        panel_label(ax, lab, "upper center")
    axes[0][0].set_ylabel("y")
    axes[0][0].legend(loc="upper center", fontsize=7.5, frameon=False)
    axes[0][0].set_ylim(-20, 90)
    fig.tight_layout()
    return _save(fig, "fig3-3", out_dir)


def fig_3_4(out_dir=None):
    """Bootstrap (top) against posterior samples (bottom) for a Bernoulli MLE, N = 10 and N = 100, theta* = 0.7."""
    rng = np.random.default_rng(4)
    S = 10000
    fig, axes = panel_grid(2, 2, size=(3.2, 2.2), sharex=True)
    lab = iter("abcd")
    datasets = {N: (rng.random(N) < 0.7).astype(int) for N in (10, 100)}
    for row in range(2):
        for col, N in enumerate((10, 100)):
            ax = axes[row][col]
            data = datasets[N]
            n1 = data.sum()
            mle = n1 / N
            if row == 0:
                draws = rng.binomial(N, mle, S) / N       # parametric bootstrap
                ttl = "Bootstrap: N = %d" % N
            else:
                draws = rng.beta(1 + n1, 1 + N - n1, S)
                ttl = "Bayes: N = %d" % N
            ax.hist(draws, bins=np.linspace(0, 1, 41), color=BLUE, alpha=0.8)
            ax.axvline(mle, color="k", lw=1)
            ax.set_title(ttl, fontsize=9)
            panel_label(ax, next(lab), "upper left")
            if row == 1:
                ax.set_xlabel(r"$\theta$")
    fig.tight_layout()
    return _save(fig, "fig3-4", out_dir)


# ============================================================================= Section 3.4: conjugate priors
def gauss_mean_post(y, sigma2, m0, tau02):
    """(3.99)-(3.100): posterior mean and variance of mu, known variance."""
    N = len(y)
    lam, lam0 = 1 / sigma2, 1 / tau02
    prec = lam0 + N * lam
    return (N * lam * np.mean(y) + lam0 * m0) / prec, 1 / prec


def fig_3_5(out_dir=None):
    """Inferring the mean of a univariate Gaussian with known variance; strong and weak prior."""
    y = np.array([1.2, 2.4, 3.0])
    x = np.linspace(-5, 5, 501)
    fig, axes = panel_grid(1, 2, size=(3.4, 2.6), sharey=True)
    for ax, v0, lab in zip(axes[0], [1.0, 5.0], "ab"):
        pm, pv = gauss_mean_post(y, 1.0, 0.0, v0)
        lik = stats.norm.pdf(x, y.mean(), np.sqrt(1.0 / len(y)))
        ax.plot(x, stats.norm.pdf(x, 0, np.sqrt(v0)), "k", label="prior")
        ax.plot(x, lik, RED, ls="--", label="likelihood")
        ax.plot(x, stats.norm.pdf(x, pm, np.sqrt(pv)), BLUE, label="posterior")
        ax.set_xlabel(r"$\mu$")
        ax.set_title("prior variance %g" % v0, fontsize=9)
        panel_label(ax, lab, "upper left")
    axes[0][0].legend(loc="upper right", fontsize=7.5, frameon=False)
    fig.tight_layout()
    return _save(fig, "fig3-5", out_dir)


def invgamma_pdf(x, a, b):
    return stats.invgamma.pdf(x, a, scale=b)


def fig_3_6(out_dir=None):
    """Sequential updating of the posterior of sigma^2 (known mean 5, true variance 10), improper prior."""
    rng = np.random.default_rng(6)
    y = rng.normal(5, np.sqrt(10), 100)
    s2 = np.linspace(0.05, 15, 600)
    fig, ax = plt.subplots(figsize=(4.4, 2.8))
    for N, c in zip((2, 5, 50, 100), (BLUE, GREEN, ORANGE, RED)):
        ss = np.sum((y[:N] - 5) ** 2)
        ax.plot(s2, invgamma_pdf(s2, N / 2, ss / 2), color=c, label="N = %d" % N)
    ax.axvline(10, color="k", ls=":", lw=1)
    ax.set_xlabel(r"$\sigma^2$")
    ax.legend(frameon=False, fontsize=8)
    ax.set_title(r"prior IG(0, 0), true $\sigma^2$ = 10", fontsize=9)
    fig.tight_layout()
    return _save(fig, "fig3-6", out_dir)


def nix_logpdf(mu, s2, m0, k0, nu0, t20):
    """Normal inverse chi-squared density (3.118)."""
    return stats.norm.logpdf(mu, m0, np.sqrt(s2 / k0)) + stats.invgamma.logpdf(s2, nu0 / 2, scale=nu0 * t20 / 2)


def nix_update(y, m0, k0, nu0, t20):
    """(3.120)-(3.124)."""
    y = np.asarray(y, float)
    N, ybar = len(y), y.mean()
    kN, nuN = k0 + N, nu0 + N
    mN = (k0 * m0 + N * ybar) / kN
    ss = nu0 * t20 + np.sum((y - ybar) ** 2) + N * k0 / kN * (m0 - ybar) ** 2
    return mN, kN, nuN, ss / nuN


def fig_3_7(out_dir=None):
    """The NIX(0, kappa, 1, 1) density, kappa = 1 and 5: contours in the (mu, sigma^2) plane."""
    mu = np.linspace(-1, 1, 201)
    s2 = np.linspace(0.15, 2.0, 201)
    M, S = np.meshgrid(mu, s2)
    fig, axes = panel_grid(1, 2, size=(3.4, 2.8), sharey=True)
    for ax, k, lab in zip(axes[0], (1, 5), "ab"):
        Z = np.exp(nix_logpdf(M, S, 0.0, k, 1.0, 1.0))
        cs = ax.contourf(M, S, Z, levels=12, cmap="Blues")
        ax.contour(M, S, Z, levels=12, colors="k", linewidths=0.3)
        ax.set_xlabel(r"$\mu$")
        ax.set_title(r"NI$\chi^2$($\mu_0$=0, $\kappa_0$=%d, $\nu_0$=1, $\sigma_0^2$=1)" % k, fontsize=8.5)
        panel_label(ax, lab, "upper right")
    axes[0][0].set_ylabel(r"$\sigma^2$")
    fig.tight_layout()
    return _save(fig, "fig3-7", out_dir)


def fig_3_8(out_dir=None):
    """Bayesian inference for a 2-d Gaussian mean with known sensor covariance (simulated data, 10 points)."""
    rng = np.random.default_rng(8)
    z = np.array([0.5, 0.5])
    Sy = 0.1 * np.array([[2.0, 1.0], [1.0, 1.0]])
    Y = rng.multivariate_normal(z, Sy, 10)
    V0 = 0.1 * np.eye(2)
    Vn = np.linalg.inv(np.linalg.inv(V0) + 10 * np.linalg.inv(Sy))
    mn = Vn @ (np.linalg.inv(Sy) @ (10 * Y.mean(0)))
    g = np.linspace(-1.2, 1.2, 241)
    G1, G2 = np.meshgrid(g, g)
    P = np.stack([G1.ravel(), G2.ravel()], 1)
    fig, axes = panel_grid(1, 3, size=(2.6, 2.6), sharex=True, sharey=True)
    axes[0][0].plot(Y[:, 0], Y[:, 1], "ro", ms=3.5)
    axes[0][0].plot(*z, "k+", ms=10)
    axes[0][0].set_title("data", fontsize=9)
    for ax, (m, V), ttl in zip(axes[0][1:], [(np.zeros(2), V0), (mn, Vn)], ["prior", "posterior after 10 points"]):
        d = stats.multivariate_normal(m, V).pdf(P).reshape(G1.shape)
        ax.contourf(G1, G2, d, levels=10, cmap="Blues")
        ax.plot(*z, "k+", ms=10)
        ax.set_title(ttl, fontsize=9)
    for ax, lab in zip(axes[0], "abc"):
        ax.set_xlabel("$y_1$ / $z_1$")
        panel_label(ax, lab, "upper right")
    fig.tight_layout()
    return _save(fig, "fig3-8", out_dir)


def fig_3_10(out_dir=None):
    """A mixture of two beta priors and its posterior after 20 heads and 10 tails."""
    w0, ab = [0.5, 0.5], [(20, 20), (30, 10)]
    wp, abp = mix_beta_posterior(w0, ab, 20, 10)
    th = np.linspace(0, 1, 501)
    mix = lambda w, p: sum(wi * beta_pdf(th, a, b) for wi, (a, b) in zip(w, p))
    fig, ax = plt.subplots(figsize=(4.4, 2.8))
    ax.plot(th, mix(w0, ab), "k", label="prior")
    ax.plot(th, mix(wp, abp), RED, ls="--", label="posterior")
    ax.set_xlabel(r"$\theta$")
    ax.legend(frameon=False)
    ax.set_title("posterior weights %.3f, %.3f" % tuple(wp), fontsize=9)
    fig.tight_layout()
    return _save(fig, "fig3-10", out_dir)


def lkj_corr_pdf(rho, eta):
    """Density of the correlation of a 2 x 2 LKJ(eta) matrix: proportional to (1 - rho^2)^(eta - 1)."""
    return (1 - rho ** 2) ** (eta - 1) / (2 ** (2 * eta - 1) * special.beta(eta, eta))


def fig_3_11(out_dir=None):
    """Distribution of the correlation induced by the 2-d LKJ(eta) prior, eta = 1, 2, 4."""
    r = np.linspace(-1, 1, 501)
    fig, ax = plt.subplots(figsize=(4.2, 2.8))
    for eta, c in zip((1, 2, 4), (BLUE, GREEN, RED)):
        ax.plot(r, lkj_corr_pdf(r, eta), color=c, label=r"$\eta$ = %d" % eta)
    ax.set_xlabel("correlation")
    ax.set_ylabel("density")
    ax.legend(frameon=False)
    fig.tight_layout()
    return _save(fig, "fig3-11", out_dir)


# ============================================================================= Section 3.5: noninformative priors
def maxent_mean_pmf(support, mean):
    """Maximum entropy pmf on a finite support with a given mean: p_c proportional to exp(lam * c)."""
    c = np.asarray(support, float)

    def f(lam):
        p = np.exp(lam * (c - c.mean()))
        p /= p.sum()
        return p @ c - mean
    lam = optimize.brentq(f, -50, 50)
    p = np.exp(lam * (c - c.mean()))
    return p / p.sum(), lam


def fig_3_12(out_dir=None):
    """Three maximum entropy priors on {1, ..., 10}."""
    c = np.arange(1, 11)
    p_mean, _ = maxent_mean_pmf(c, 1.5)
    p34 = np.full(10, 0.2 / 8)
    p34[[2, 3]] = 0.4
    fig, ax = plt.subplots(figsize=(4.6, 2.8))
    ax.plot(c, np.full(10, 0.1), "k-o", ms=3.5, label="unconstrained")
    ax.plot(c, p_mean, RED, marker="s", ms=3.5, ls="--", label="mean 1.5")
    ax.plot(c, p34, BLUE, marker="^", ms=3.5, ls=":", label="p(3,4) = 0.8")
    ax.set_xlabel(r"$\theta$")
    ax.set_ylabel(r"$p(\theta)$")
    ax.set_xticks(c)
    ax.legend(frameon=False, fontsize=8)
    fig.tight_layout()
    return _save(fig, "fig3-12", out_dir)


def fig_3_13(out_dir=None):
    """Jeffreys prior invariance: Alice uses the rate theta, Bob the odds phi = theta/(1 - theta)."""
    n, x = 10, 3
    th = np.linspace(0.001, 0.999, 999)
    ph = np.linspace(0.005, 10, 1000)
    pa = beta_pdf(th, 0.5, 0.5)
    pb = ph ** -0.5 / (1 + ph) / np.pi
    post_a = beta_pdf(th, 0.5 + x, 0.5 + n - x)
    # Bob's posterior: p(phi|x) proportional to phi^(x - 1/2) (1 + phi)^(-n - 1)
    post_b = ph ** (x - 0.5) * (1 + ph) ** (-n - 1)
    post_b /= np.trapz(post_b, ph)
    post_a_in_phi = beta_pdf(ph / (1 + ph), 0.5 + x, 0.5 + n - x) / (1 + ph) ** 2
    fig, axes = panel_grid(2, 2, size=(3.2, 2.2))
    axes[0][0].plot(th, pa, "k"); axes[0][0].set_title("Jeffreys prior, Alice ($\\theta$)", fontsize=8.5); axes[0][0].set_ylim(0, 5)
    axes[0][1].plot(ph, pb, "k"); axes[0][1].set_title("Jeffreys prior, Bob ($\\phi$)", fontsize=8.5); axes[0][1].set_ylim(0, 1.5)
    axes[1][0].plot(th, post_a, BLUE); axes[1][0].set_title("posterior, Alice", fontsize=8.5); axes[1][0].set_xlabel(r"$\theta$")
    axes[1][1].plot(ph, post_b, BLUE, label="Bob directly")
    axes[1][1].plot(ph, post_a_in_phi, RED, ls="--", label=r"Alice's, times $|d\theta/d\phi|$")
    axes[1][1].set_xlabel(r"$\phi$")
    axes[1][1].legend(frameon=False, fontsize=7)
    axes[1][1].set_title("posterior, Bob", fontsize=8.5)
    for ax, l in zip(axes.ravel(), "abcd"):
        panel_label(ax, l, "upper right")
    fig.tight_layout()
    return _save(fig, "fig3-13", out_dir)


# ============================================================================= Section 3.6-3.7: hierarchical models and EB
def hdi_from_samples(s, mass=0.95):
    """Shortest interval containing a fraction `mass` of the samples."""
    s = np.sort(np.asarray(s))
    k = int(np.ceil(mass * len(s)))
    w = s[k - 1:] - s[:len(s) - k + 1]
    i = int(np.argmin(w))
    return s[i], s[i + k - 1]


def rats_ab_logpost(a, b, y=RATS_Y, n=RATS_N):
    """log p(a, b | D) up to a constant for the hierarchical binomial model with p(a, b) proportional to (a+b)^(-5/2),
    after integrating out every theta_j analytically; the Jacobian |d(a,b)/d(u,v)| = a b is NOT included."""
    a, b = np.asarray(a)[..., None], np.asarray(b)[..., None]
    ll = special.betaln(a + y, b + n - y) - special.betaln(a, b)
    return -2.5 * np.log((a + b)[..., 0]) + ll.sum(-1)


def rats_grid_posterior(S=4000, seed=15, ng=260):
    """Exact posterior sampling for the rats model: a grid over (u, v) = (log(a/b), log(a+b)) for the hyperparameters,
    then theta_j | a, b, D ~ Beta(a + y_j, b + n_j - y_j). Returns hyperparameter and theta samples."""
    rng = np.random.default_rng(seed)
    u = np.linspace(-3.2, 0.2, ng)
    v = np.linspace(-0.5, 5.5, ng)
    U, V = np.meshgrid(u, v, indexing="ij")
    a = np.exp(V) / (1 + np.exp(-U))
    b = np.exp(V) / (1 + np.exp(U))
    lp = rats_ab_logpost(a, b) + np.log(a) + np.log(b)
    p = np.exp(lp - lp.max()).ravel()
    p /= p.sum()
    idx = rng.choice(p.size, S, p=p)
    du, dv = u[1] - u[0], v[1] - v[0]
    ui = U.ravel()[idx] + rng.uniform(-du / 2, du / 2, S)
    vi = V.ravel()[idx] + rng.uniform(-dv / 2, dv / 2, S)
    a_s = np.exp(vi) / (1 + np.exp(-ui))
    b_s = np.exp(vi) / (1 + np.exp(ui))
    theta = rng.beta(a_s[:, None] + RATS_Y, b_s[:, None] + RATS_N - RATS_Y)
    return a_s, b_s, theta


def beta_eb_fit(y=RATS_Y, n=RATS_N):
    """Type II maximum likelihood for (a, b) in the hierarchical binomial model, (3.260)-(3.262)."""
    f = lambda t: -np.sum(special.betaln(np.exp(t[0]) + y, np.exp(t[1]) + n - y) - special.betaln(np.exp(t[0]), np.exp(t[1])))
    r = optimize.minimize(f, np.log([1.0, 5.0]), method="Nelder-Mead", options={"xatol": 1e-9, "fatol": 1e-12, "maxiter": 2000})
    return np.exp(r.x), -r.fun


def fig_3_15(out_dir=None):
    """The rat tumour data and the hierarchical binomial posterior (exact posterior sampling instead of HMC)."""
    a_s, b_s, theta = rats_grid_posterior()
    mle = RATS_Y / RATS_N
    pooled = RATS_Y.sum() / RATS_N.sum()
    pm = theta.mean(0)
    pop = (a_s / (a_s + b_s)).mean()
    J = len(RATS_Y)
    fig = plt.figure(figsize=(7.0, 4.6))
    gs = fig.add_gridspec(4, 2, width_ratios=[1.6, 1], hspace=0.35, wspace=0.3)
    x = np.arange(J)
    ax = fig.add_subplot(gs[0, 0]); ax.bar(x, RATS_Y, color=GREY); ax.set_ylabel("positives", fontsize=8)
    ax = fig.add_subplot(gs[1, 0], sharex=ax); ax.bar(x, RATS_N, color=GREY); ax.set_ylabel("group size", fontsize=8)
    ax = fig.add_subplot(gs[2, 0], sharex=ax); ax.bar(x, mle, color=BLUE); ax.axhline(pooled, color=RED); ax.set_ylabel(r"$\hat\theta_{MLE}$", fontsize=8)
    ax = fig.add_subplot(gs[3, 0], sharex=ax); ax.bar(x, pm, color=GREEN); ax.axhline(pop, color=RED); ax.set_ylabel("post. mean", fontsize=8)
    ax.set_xlabel("type of rat tumour")
    axb = fig.add_subplot(gs[:, 1])
    for j in range(J):
        lo, hi = hdi_from_samples(theta[:, j])
        axb.plot([lo, hi], [j, j], color=BLUE, lw=1.2)
        axb.plot(pm[j], j, "k.", ms=2.5)
    axb.axvline(pop, color=RED, lw=1)
    axb.set_ylim(-1, J)
    axb.set_xlabel(r"$\theta$ (95% HDI)")
    axb.set_yticks(range(0, J, 10))
    for a_, l in zip([fig.axes[0], axb], "ab"):
        panel_label(a_, l, "upper right")
    return _save(fig, "fig3-15", out_dir)


def fig_3_20(out_dir=None):
    """Empirical Bayes for the rats data: type II ML for (a, b), then conjugate posteriors."""
    (a, b), _ = beta_eb_fit()
    mle = RATS_Y / RATS_N
    pm = (a + RATS_Y) / (a + b + RATS_N)
    lo = stats.beta.ppf(0.025, a + RATS_Y, b + RATS_N - RATS_Y)
    hi = stats.beta.ppf(0.975, a + RATS_Y, b + RATS_N - RATS_Y)
    J = len(RATS_Y)
    fig, axes = panel_grid(1, 2, size=(3.5, 4.0))
    ax = axes[0][0]
    ax.plot(mle, np.arange(J), "o", color=BLUE, ms=3, label="MLE")
    ax.plot(pm, np.arange(J), "s", color=GREEN, ms=3, label="posterior mean")
    ax.axvline(RATS_Y.sum() / RATS_N.sum(), color=RED, lw=1, label="pooled MLE")
    ax.set_xlabel(r"$\theta$"); ax.set_ylabel("type of rat tumour"); ax.legend(frameon=False, fontsize=7)
    ax = axes[0][1]
    for j in range(J):
        ax.plot([lo[j], hi[j]], [j, j], color=BLUE, lw=1.2)
    ax.axvline(a / (a + b), color=RED, lw=1)
    ax.set_xlabel(r"$\theta$ (95% interval)"); ax.set_title("EB: a = %.2f, b = %.2f" % (a, b), fontsize=8.5)
    for a_, l in zip(axes[0], "ab"):
        panel_label(a_, l, "lower right")
    fig.tight_layout()
    return _save(fig, "fig3-20", out_dir)


def schools_tau_posterior(y=SCHOOLS_Y, s=SCHOOLS_S, tau=None, prior="halfcauchy", m0=0.0, s0=5.0, scale=5.0):
    """Marginal posterior p(tau | D) on a grid for the eight-schools model theta_j ~ N(mu, tau^2), mu ~ N(m0, s0^2)
    (s0 = inf for a flat prior on mu), by integrating out mu and every theta_j analytically.
    prior: 'halfcauchy' (scale), 'flat' or 'halfnormal'. Returns (tau grid, normalised density)."""
    if tau is None:
        tau = np.linspace(0, 40, 4001)[1:]
    lp = np.empty(len(tau))
    one = np.ones(len(y))
    for i, t in enumerate(tau):
        d = s ** 2 + t ** 2
        if np.isinf(s0):
            w = 1 / d
            mh = (w * y).sum() / w.sum()
            lp[i] = -0.5 * np.log(d).sum() - 0.5 * np.log(w.sum()) - 0.5 * np.sum((y - mh) ** 2 * w)
        else:
            C = np.diag(d) + s0 ** 2 * np.outer(one, one)
            lp[i] = stats.multivariate_normal.logpdf(y, m0 * one, C)
    if prior == "halfcauchy":
        lp += stats.halfcauchy.logpdf(tau, scale=scale)
    elif prior == "halfnormal":
        lp += stats.halfnorm.logpdf(tau, scale=scale)
    p = np.exp(lp - lp.max())
    return tau, p / np.trapz(p, tau)


def schools_sample(S=20000, seed=8, **kw):
    """Exact joint posterior samples (tau, mu, theta_1..8) for the eight-schools model."""
    rng = np.random.default_rng(seed)
    tau, p = schools_tau_posterior(**kw)
    cdf = np.cumsum(p); cdf /= cdf[-1]
    t = np.interp(rng.random(S), cdf, tau)
    y, s = SCHOOLS_Y, SCHOOLS_S
    m0, s0 = kw.get("m0", 0.0), kw.get("s0", 5.0)
    mu = np.empty(S)
    th = np.empty((S, len(y)))
    for i, ti in enumerate(t):
        w = 1 / (s ** 2 + ti ** 2)
        prec = w.sum() + (0 if np.isinf(s0) else 1 / s0 ** 2)
        mean = ((w * y).sum() + (0 if np.isinf(s0) else m0 / s0 ** 2)) / prec
        mu[i] = rng.normal(mean, 1 / np.sqrt(prec))
        pv = 1 / (1 / s ** 2 + 1 / ti ** 2)
        pmn = pv * (y / s ** 2 + mu[i] / ti ** 2)
        th[i] = rng.normal(pmn, np.sqrt(pv))
    return t, mu, th


def fig_3_16(out_dir=None):
    """The eight-schools data and posterior intervals of the school effects."""
    t, mu, th = schools_sample()
    fig, axes = panel_grid(1, 2, size=(3.5, 3.0), sharey=True)
    y, s = SCHOOLS_Y, SCHOOLS_S
    pooled = np.sum(y / s ** 2) / np.sum(1 / s ** 2)
    ax = axes[0][0]
    ax.errorbar(y, np.arange(8), xerr=s, fmt="o", color=BLUE, ms=4, capsize=2)
    ax.axvline(pooled, color="k", lw=1)
    ax.set_yticks(range(8)); ax.invert_yaxis(); ax.set_xlabel(r"$y_j \pm \sigma_j$"); ax.set_ylabel("school")
    ax = axes[0][1]
    for j in range(8):
        lo, hi = np.percentile(th[:, j], [2.5, 97.5])
        ax.plot([lo, hi], [j, j], color=BLUE, lw=1.6)
        ax.plot(th[:, j].mean(), j, "ko", ms=3)
    ax.axvline(mu.mean(), color="k", lw=1)
    ax.set_xlabel(r"$\theta_j$ (95% interval)")
    for a_, l in zip(axes[0], "ab"):
        panel_label(a_, l, "lower right")
    fig.tight_layout()
    return _save(fig, "fig3-16", out_dir)


def fig_3_17(out_dir=None):
    """Marginal posterior of tau in the eight-schools model."""
    tau, p = schools_tau_posterior()
    t, _, _ = schools_sample()
    lo, hi = hdi_from_samples(t)
    fig, ax = plt.subplots(figsize=(4.4, 2.8))
    ax.plot(tau, p, "k")
    m = (tau >= lo) & (tau <= hi)
    ax.fill_between(tau[m], 0, p[m], color=BLUE, alpha=0.3)
    ax.axvline(t.mean(), color=RED, ls="--", lw=1)
    ax.set_xlim(0, 25)
    ax.set_xlabel(r"$\tau$")
    ax.set_title("posterior mean %.1f, 95%% HPD (%.2f, %.1f)" % (t.mean(), lo, hi), fontsize=8.5)
    fig.tight_layout()
    return _save(fig, "fig3-17", out_dir)


def fig_3_18(out_dir=None):
    """The funnel of the centred parameterisation and the non-centred coordinates (exact posterior draws)."""
    t, mu, th = schools_sample(S=6000)
    eta = (th - mu[:, None]) / t[:, None]
    fig, axes = panel_grid(1, 2, size=(3.4, 2.8), sharey=True)
    axes[0][0].plot(th[:, 0], np.log(t), ".", ms=1.5, color=BLUE, alpha=0.4)
    axes[0][0].set_xlabel(r"$\theta_0$"); axes[0][0].set_ylabel(r"$\log\tau$"); axes[0][0].set_title("centred", fontsize=9)
    axes[0][1].plot(eta[:, 0], np.log(t), ".", ms=1.5, color=GREEN, alpha=0.4)
    axes[0][1].set_xlabel(r"$\eta_0$"); axes[0][1].set_title("non-centred", fontsize=9)
    for a_, l in zip(axes[0], "ab"):
        panel_label(a_, l, "upper right")
    fig.tight_layout()
    return _save(fig, "fig3-18", out_dir)


# ============================================================================= Section 3.8: model selection
def fig_3_22(out_dir=None):
    """Bayesian Occam's razor, computed for a coin: p(D | M) over all outcomes of N = 10 tosses (counts of heads)."""
    N = 10
    k = np.arange(N + 1)
    ps = {"M0: fair coin": stats.binom.pmf(k, N, 0.5),
          "M1: Beta(1, 1) prior": stats.betabinom.pmf(k, N, 1, 1),
          "M2: Beta(20, 20) prior": stats.betabinom.pmf(k, N, 20, 20)}
    fig, ax = plt.subplots(figsize=(4.8, 2.8))
    for (nm, p), c in zip(ps.items(), (BLUE, GREEN, RED)):
        ax.plot(k, p, "-o", color=c, ms=3.5, label=nm)
    ax.set_xlabel("number of heads, $D$"); ax.set_ylabel("$p(D\\mid M)$")
    ax.legend(frameon=False, fontsize=8)
    fig.tight_layout()
    return _save(fig, "fig3-22", out_dir)


def bic_logml_coin(n1, n0):
    """BIC approximation to log p(D | M1) for the Bernoulli model (one parameter), (3.312)."""
    N = n1 + n0
    ll = (n1 * np.log(n1 / N) if n1 else 0.0) + (n0 * np.log(n0 / N) if n0 else 0.0)
    return ll - 0.5 * np.log(N)


def fig_3_23(out_dir=None):
    """Coin tossing model selection with N = 5: log10 marginal likelihood of M1 against the number of heads, and BIC."""
    N = 5
    n1 = np.arange(N + 1)
    exact = np.array([beta_logml(k, N - k, 1, 1) for k in n1]) / np.log(10)
    bic = np.array([bic_logml_coin(k, N - k) for k in n1]) / np.log(10)
    fair = N * np.log(0.5) / np.log(10)
    fig, axes = panel_grid(1, 2, size=(3.5, 2.7))
    for ax, v, ttl, lab in zip(axes[0], (exact, bic), (r"$\log_{10}p(D|M_1)$", r"BIC approximation to $\log_{10}p(D|M_1)$"), "ab"):
        ax.plot(n1, v, "o-", color=BLUE)
        ax.axhline(fair, color=RED, ls="--", lw=1, label=r"$M_0$ (fair coin)")
        ax.set_xlabel("number of heads"); ax.set_title(ttl, fontsize=8.5)
        panel_label(ax, lab, "upper center")
    axes[0][0].legend(frameon=False, fontsize=7.5, loc="lower center")
    fig.tight_layout()
    return _save(fig, "fig3-23", out_dir)


# ============================================================================= Section 3.9: model checking
def newcomb_ppc(S=1000, seed=25):
    """Plug-in posterior predictive replications of Newcomb's data under a Gaussian model."""
    rng = np.random.default_rng(seed)
    mu, sd = NEWCOMB.mean(), NEWCOMB.std()
    rep = rng.normal(mu, sd, (S, len(NEWCOMB)))
    return rep


def fig_3_25(out_dir=None):
    """Posterior predictive check of a Gaussian model for Newcomb's speed of light data."""
    rep = newcomb_ppc()
    fig = plt.figure(figsize=(7.0, 4.4))
    gs = fig.add_gridspec(2, 3, hspace=0.45, wspace=0.35)
    ax = fig.add_subplot(gs[0, 0]); ax.hist(NEWCOMB, bins=20, color=BLUE, edgecolor="k", lw=0.4); ax.set_title("Newcomb's data", fontsize=9); panel_label(ax, "a", "upper left")
    ax = fig.add_subplot(gs[0, 1:]); 
    for i in range(3):
        ax.hist(rep[i], bins=np.linspace(-50, 60, 23), alpha=0.5, label="replication %d" % (i + 1))
    ax.set_title("data replicated from the model", fontsize=9); ax.legend(frameon=False, fontsize=7); panel_label(ax, "b", "upper left")
    mn = rep.min(1)
    ax = fig.add_subplot(gs[1, :2]); ax.hist(mn, bins=30, color=GREY); ax.axvline(NEWCOMB.min(), color=RED, lw=1.6)
    ax.set_title("test statistic min: observed %d, $p_B$ = %.3f" % (NEWCOMB.min(), np.mean(mn <= NEWCOMB.min())), fontsize=8.5); panel_label(ax, "c", "upper left")
    vr = rep.var(1)
    ax = fig.add_subplot(gs[1, 2]); ax.hist(vr, bins=30, color=GREY); ax.axvline(NEWCOMB.var(), color=RED, lw=1.6)
    ax.set_title("variance: observed %.1f, $p_B$ = %.2f" % (NEWCOMB.var(), np.mean(vr >= NEWCOMB.var())), fontsize=8.5); panel_label(ax, "d", "upper left")
    return _save(fig, "fig3-25", out_dir)


# ============================================================================= Section 3.10: hypothesis testing
def welch_diff_samples(y1, y2, S=40000, seed=27):
    """Posterior samples of mu2 - mu1 for two Gaussian groups with separate variances and the Jeffreys prior
    p(mu, sigma^2) proportional to 1/sigma^2 in each group: mu_j | D ~ t_(n_j - 1)(ybar_j, s_j^2 / n_j), independent."""
    rng = np.random.default_rng(seed)
    d = []
    for y in (y1, y2):
        n = len(y)
        d.append(y.mean() + y.std(ddof=1) / np.sqrt(n) * rng.standard_t(n - 1, S))
    return d[1] - d[0]


def fig_3_27(out_dir=None):
    """Bayesian two-sample t-test on simulated data (two groups)."""
    rng = np.random.default_rng(27)
    ya = rng.normal(1.0, 1.1, 18)
    yb = rng.normal(2.7, 1.5, 22)
    d = welch_diff_samples(ya, yb)
    lo, hi = hdi_from_samples(d)
    fig, axes = panel_grid(1, 2, size=(3.4, 2.8))
    ax = axes[0][0]
    ax.plot(np.zeros_like(ya) + rng.uniform(-0.12, 0.12, ya.size), ya, "o", color=BLUE, ms=3.5)
    ax.plot(1 + rng.uniform(-0.12, 0.12, yb.size), yb, "o", color=RED, ms=3.5)
    ax.set_xticks([0, 1]); ax.set_xticklabels(["a", "b"]); ax.set_ylabel("value"); ax.set_xlim(-0.6, 1.6)
    ax = axes[0][1]
    ax.hist(d, bins=60, density=True, color=GREY)
    ax.plot([lo, hi], [0.02, 0.02], "k", lw=3)
    ax.axvline(0, color=RED, lw=1)
    ax.set_xlabel(r"$\mu_b-\mu_a$"); ax.set_title("mean %.2f, 95%% HDI (%.2f, %.2f)" % (d.mean(), lo, hi), fontsize=8.5)
    for a_, l in zip(axes[0], "ab"):
        panel_label(a_, l, "upper left")
    fig.tight_layout()
    return _save(fig, "fig3-27", out_dir)


def fig_3_28(out_dir=None):
    """A linear regression on raw data (Pearson) and on rank-transformed data (Spearman); simulated data."""
    rng = np.random.default_rng(28)
    x = rng.normal(0, 1.8, 30)
    y = 0.8 * x + rng.normal(0, 1.2, 30)
    rx, ry = stats.rankdata(x), stats.rankdata(y)
    fig, axes = panel_grid(1, 2, size=(3.4, 3.0))
    for ax, (u, v), ttl, lab in zip(axes[0], [(x, y), (rx, ry)], ["Pearson", "Spearman"], "ab"):
        b1, b0 = np.polyfit(u, v, 1)
        ax.plot(u, v, "o", color=BLUE, ms=3.5)
        g = np.linspace(u.min(), u.max(), 10)
        ax.plot(g, b0 + b1 * g, RED)
        ax.set_title("%s: $\\beta_0$ = %.2f, $\\beta_1$ = %.2f" % (ttl, b0, b1), fontsize=8.5)
        panel_label(ax, lab, "lower right")
    fig.tight_layout()
    return _save(fig, "fig3-28", out_dir)


def ols_t_posterior(X, y):
    """Reference-prior posterior of the regression weights: Student t with N - D degrees of freedom,
    location the OLS estimate and scale matrix s^2 (X'X)^{-1}."""
    N, D = X.shape
    G = np.linalg.inv(X.T @ X)
    w = G @ X.T @ y
    s2 = np.sum((y - X @ w) ** 2) / (N - D)
    return w, s2 * G, N - D


def fig_3_29(out_dir=None):
    """One-way ANOVA with four groups as a linear model with a dummy encoding (simulated data)."""
    rng = np.random.default_rng(29)
    J, n = 4, 15
    mus = [0.0, 0.2, 1.1, 1.9]
    g = np.repeat(np.arange(J), n)
    y = np.array(mus)[g] + rng.normal(0, 1.0, J * n)
    X = np.zeros((J * n, J)); X[:, 0] = 1
    for j in range(1, J):
        X[g == j, j] = (g == j)[g == j]
    w, Sc, nu = ols_t_posterior(X, y)
    xs = np.linspace(-1.5, 3.5, 501)
    fig, ax = plt.subplots(figsize=(4.8, 2.9))
    cols = [BLUE, RED, GREEN, ORANGE]
    for j in range(J):
        lab = r"$\beta_0$ (group 1 mean)" if j == 0 else r"$\beta_%d$ (difference to $\beta_0$)" % (j + 1)
        ax.plot(xs, stats.t.pdf(xs, nu, w[j], np.sqrt(Sc[j, j])), color=cols[j], label=lab)
    ax.axvline(0, color="k", lw=0.8, ls=":")
    ax.set_xlabel("coefficient"); ax.legend(frameon=False, fontsize=7.5)
    fig.tight_layout()
    return _save(fig, "fig3-29", out_dir)


# ============================================================================= Supplement 3.1: the number game
NUMS = np.arange(1, 101)


def number_game_rules():
    """The 32 rule-like hypotheses of the number game as boolean masks over 1..100, with their prior weights
    (even and odd are four times as likely as an ordinary rule, the two 'unnatural' concepts a tenth)."""
    H, w = {}, {}
    H["even"], H["odd"] = NUMS % 2 == 0, NUMS % 2 == 1
    H["squares"] = np.isin(NUMS, [k * k for k in range(1, 11)])
    for m in range(3, 11):
        H["mult of %d" % m] = NUMS % m == 0
    for d in range(1, 10):
        H["ends in %d" % d] = NUMS % 10 == d
    for p in range(2, 11):
        H["powers of %d" % p] = np.isin(NUMS, [p ** k for k in range(1, 8) if p ** k <= 100])
    H["all"] = np.ones(100, bool)
    H["powers of 2 +{37}"] = H["powers of 2"] | (NUMS == 37)
    H["powers of 2 -{32}"] = H["powers of 2"] & (NUMS != 32)
    for k in H:
        w[k] = 4.0 if k in ("even", "odd") else (0.1 if k.startswith("powers of 2 ") else 1.0)
    tot = sum(w.values())
    return H, {k: v / tot for k, v in w.items()}


def number_game_posterior(D, rules_only=True, pi=0.5):
    """Posterior over hypotheses with the size principle (3.1): p(D|h) = size(h)^(-N) 1[D in h].
    Returns (names, masks, prior, likelihood, posterior); with rules_only=False the 5050 intervals are added,
    prior pi for the rule family and 1 - pi spread uniformly over intervals, (3.2)."""
    H, w = number_game_rules()
    names, masks, prior = list(H), [H[k] for k in H], np.array([w[k] for k in H])
    if not rules_only:
        prior = prior * pi
        for n in range(1, 101):
            for m in range(n, 101):
                names.append("[%d, %d]" % (n, m)); masks.append((NUMS >= n) & (NUMS <= m))
        prior = np.concatenate([prior, np.full(len(names) - len(prior), (1 - pi) / 5050)])
    M = np.array(masks)
    D = np.asarray(D)
    size = M.sum(1)
    inside = M[:, D - 1].all(1)
    lik = np.where(inside, size.astype(float) ** (-len(D)), 0.0)
    post = prior * lik
    return names, M, prior, lik, post / post.sum()


def fig_S3_2(out_dir=None):
    """Posterior membership probabilities in the number game with the full hypothesis space (rules and intervals)."""
    sets = [[16], [60], [16, 8, 2, 64], [16, 23, 19, 20]]
    fig, axes = panel_grid(4, 1, size=(5.6, 1.35), sharex=True)
    for ax, D in zip(axes[:, 0], sets):
        names, M, prior, lik, post = number_game_posterior(D, rules_only=False)
        mem = post @ M
        ax.bar(NUMS, mem, color=BLUE, width=0.8)
        ax.set_ylim(0, 1.05); ax.set_ylabel("p(y in C | D)", fontsize=7)
        ax.text(0.99, 0.82, "D = {%s}" % ", ".join(map(str, D)), transform=ax.transAxes, ha="right", fontsize=8)
    axes[-1][0].set_xlabel("y"); axes[-1][0].set_xticks(range(4, 101, 8))
    fig.tight_layout()
    return _save(fig, "figS3-2", out_dir)


def fig_S3_3(out_dir=None):
    """Prior, likelihood and posterior over the rule hypotheses, D = {16} and D = {2, 8, 16, 64}."""
    fig, axes = panel_grid(1, 2, size=(4.4, 6.2))
    for ax, D in zip(axes[0], [[16], [2, 8, 16, 64]]):
        names, M, prior, lik, post = number_game_posterior(D)
        y = np.arange(len(names))
        sc = lik / lik.max()
        ax.barh(y + 0.27, prior / prior.max(), height=0.27, color=GREY, label="prior")
        ax.barh(y, sc, height=0.27, color=RED, label="likelihood (rescaled)")
        ax.barh(y - 0.27, post / post.max(), height=0.27, color=BLUE, label="posterior (rescaled)")
        ax.set_yticks(y); ax.set_yticklabels(names, fontsize=5.5); ax.invert_yaxis()
        ax.set_title("D = {%s}" % ", ".join(map(str, D)), fontsize=9)
    axes[0][0].legend(frameon=False, fontsize=6.5, loc="lower right")
    for a_, l in zip(axes[0], "ab"):
        panel_label(a_, l, "upper right")
    fig.tight_layout()
    return _save(fig, "figS3-3", out_dir)


def fig_S3_4(out_dir=None):
    """Posterior over rules and the induced membership probability after D = {16}."""
    names, M, prior, lik, post = number_game_posterior([16])
    order = np.argsort(-post)[:9]
    fig = plt.figure(figsize=(6.0, 4.0))
    gs = fig.add_gridspec(2, 2, width_ratios=[4, 1], height_ratios=[1, 2.4], hspace=0.08, wspace=0.05)
    ax0 = fig.add_subplot(gs[0, 0]); ax0.bar(NUMS, post @ M, color=BLUE); ax0.set_ylim(0, 1); ax0.set_ylabel("p(y in C | D)", fontsize=8)
    ax1 = fig.add_subplot(gs[1, 0], sharex=ax0)
    for r, i in enumerate(order[::-1]):
        xs = NUMS[M[i]]
        ax1.plot(xs, np.full(xs.size, r), "ko", ms=2.2)
    ax1.set_yticks(range(len(order))); ax1.set_yticklabels([names[i] for i in order[::-1]], fontsize=7)
    ax1.set_xlabel("y")
    ax2 = fig.add_subplot(gs[1, 1], sharey=ax1); ax2.barh(range(len(order)), post[order[::-1]], color=RED); ax2.set_xlabel("p(h | D)", fontsize=8)
    plt.setp(ax2.get_yticklabels(), visible=False); plt.setp(ax0.get_xticklabels(), visible=False)
    return _save(fig, "figS3-4", out_dir)


# ============================================================================= Supplement 3.1.2: the healthy levels game
def interval_posterior_samples(y, S, rng):
    """Exact samples from p(l, s | y) proportional to s^(-N-1) on {l <= min y, l + s >= max y} (3.9) with p(s) ~ 1/s:
    with a = min y - l and b = l + s - max y, the sum t = a + b has density proportional to t (r + t)^(-N-1) and
    a | t is uniform on (0, t)."""
    y = np.asarray(y)
    N, r = len(y), y.max() - y.min()
    u = rng.random(S)
    G0 = r ** (-(N - 1)) / (N * (N - 1))
    # survival function of t, solved by bisection in log t
    lo, hi = np.full(S, -30.0), np.full(S, 30.0)
    for _ in range(80):
        mid = (lo + hi) / 2
        t = r * np.exp(mid)
        surv = ((r + t) ** (-(N - 1)) / (N - 1) - r * (r + t) ** (-N) / N) / G0
        big = surv > u            # survival still above u: need larger t
        lo = np.where(big, mid, lo)
        hi = np.where(big, hi, mid)
    t = r * np.exp((lo + hi) / 2)
    a = t * rng.random(S)
    return y.min() - a, r + t


def fig_S3_5(out_dir=None):
    """Samples of rectangles from the posterior of the healthy levels game, N = 3 and N = 12 (simulated points)."""
    rng = np.random.default_rng(35)
    fig, axes = panel_grid(1, 2, size=(3.4, 3.2), sharex=True, sharey=True)
    for ax, N, lab in zip(axes[0], (3, 12), "ab"):
        pts = np.column_stack([rng.uniform(0.35, 0.55, N), rng.uniform(0.45, 0.7, N)])
        for _ in range(25):
            l1, s1 = interval_posterior_samples(pts[:, 0], 1, rng)
            l2, s2 = interval_posterior_samples(pts[:, 1], 1, rng)
            ax.add_patch(plt.Rectangle((l1[0], l2[0]), s1[0], s2[0], fill=False, ec=BLUE, lw=0.6, alpha=0.7))
        ax.plot(pts[:, 0], pts[:, 1], "rx", ms=6)
        ax.set_xlim(0.0, 1.0); ax.set_ylim(0.0, 1.0)
        ax.set_title("N = %d" % N, fontsize=9); panel_label(ax, lab, "upper right")
    fig.tight_layout()
    return _save(fig, "figS3-5", out_dir)


def healthy_predictive(pts, grid):
    """(3.12): p(y in rectangle | D) for the grid of points (rows)."""
    N = len(pts)
    p = np.ones(len(grid))
    for j in range(2):
        lo, hi = pts[:, j].min(), pts[:, j].max()
        d = np.maximum(0, np.maximum(lo - grid[:, j], grid[:, j] - hi))
        p *= (1 + d / (hi - lo)) ** (-(N - 1))
    return p


def fig_S3_6(out_dir=None):
    """Bayesian predictive (3.12) against the MLE plug-in (smallest enclosing rectangle), N = 3 and N = 12."""
    rng = np.random.default_rng(35)
    g = np.linspace(0, 1, 201)
    G1, G2 = np.meshgrid(g, g)
    P = np.column_stack([G1.ravel(), G2.ravel()])
    fig, axes = panel_grid(2, 2, size=(2.9, 2.8), sharex=True, sharey=True)
    for col, N in enumerate((3, 12)):
        pts = np.column_stack([rng.uniform(0.35, 0.55, N), rng.uniform(0.45, 0.7, N)])
        bay = healthy_predictive(pts, P).reshape(G1.shape)
        mle = ((P >= pts.min(0)) & (P <= pts.max(0))).all(1).reshape(G1.shape).astype(float)
        for row, (Z, nm) in enumerate(((bay, "Bayes"), (mle, "MLE plug-in"))):
            ax = axes[row][col]
            ax.contourf(G1, G2, Z, levels=np.linspace(0, 1, 11), cmap="Blues")
            ax.plot(pts[:, 0], pts[:, 1], "rx", ms=5)
            ax.set_title("%s, N = %d" % (nm, N), fontsize=8.5)
    for ax, l in zip(axes.ravel(), "abcd"):
        panel_label(ax, l, "upper right")
    fig.tight_layout()
    return _save(fig, "figS3-6", out_dir)


# ============================================================================= Supplement 3.2: informative priors
def taxi_median(t, prior_pdf, grid):
    """Posterior median of T given one observation t, likelihood 1/T on T >= t, prior pdf on a grid (3.13)-(3.15)."""
    out = []
    for ti in np.atleast_1d(t):
        m = grid >= ti
        dens = prior_pdf(grid[m]) / grid[m]
        c = np.concatenate([[0], np.cumsum(0.5 * (dens[1:] + dens[:-1]) * np.diff(grid[m]))])
        out.append(np.interp(0.5 * c[-1], c, grid[m]))
    return np.array(out)


def fig_S3_8(out_dir=None):
    """Priors (top) and the induced posterior medians of T given t (bottom): Gaussian, power law, Erlang."""
    T = np.linspace(0.05, 400, 40000)
    t = np.linspace(1, 60, 60)
    fig, axes = panel_grid(2, 3, size=(2.8, 2.4))
    cols = (BLUE, RED, GREEN)
    gauss = [(30, 10), (25, 10), (15, 10)]
    for (m, s), c in zip(gauss, cols):
        axes[0][0].plot(T[T < 100], stats.norm.pdf(T[T < 100], m, s), color=c, label=r"$\mu$=%d" % m)
        axes[1][0].plot(t, taxi_median(t, lambda x, m=m, s=s: stats.norm.pdf(x, m, s), T), color=c)
    axes[0][0].set_title("Gaussian prior", fontsize=9)
    for g, c in zip((1, 1.5, 2), cols):
        x = T[(T >= 1) & (T < 100)]
        axes[0][1].plot(x, x ** -g, color=c, label=r"$\gamma$=%g" % g)
        axes[1][1].plot(t, taxi_median(t, lambda x, g=g: x ** -g, T), color=c)
        axes[1][1].plot(t, 2 ** (1 / g) * t, color=c, ls=":", lw=0.8)
    axes[0][1].set_title("power-law prior (unnormalised)", fontsize=9)
    axes[0][1].set_xscale("log"); axes[0][1].set_yscale("log")
    for b, c in zip((30, 18, 10), cols):
        axes[0][2].plot(T[T < 100], stats.gamma.pdf(T[T < 100], 2, scale=b), color=c, label=r"$\beta$=%d" % b)
        axes[1][2].plot(t, taxi_median(t, lambda x, b=b: x * np.exp(-x / b), T), color=c)
        axes[1][2].plot(t, t + b * np.log(2), color=c, ls=":", lw=0.8)
    axes[0][2].set_title("Erlang prior", fontsize=9)
    for ax in axes[0]:
        ax.set_xlabel(r"$T$"); ax.legend(frameon=False, fontsize=7)
    for ax in axes[1]:
        ax.set_xlabel("$t$"); ax.plot(t, t, "k", lw=0.5)
    axes[1][0].set_ylabel(r"predicted $T$")
    fig.tight_layout()
    return _save(fig, "figS3-8", out_dir)


# ============================================================================= Supplement 3.3: Tweedie's formula
def fig_S3_9(out_dir=None):
    """Tweedie's formula with a kernel estimate of the marginal score against the exact posterior mean (not in the book)."""
    rng = np.random.default_rng(39)
    n = 20000
    comp = rng.random(n) < 0.5
    mu = np.where(comp, rng.normal(-2, 0.5, n), rng.normal(1.5, 1.0, n))
    y = mu + rng.normal(0, 1, n)
    g = np.linspace(-6, 6, 241)
    h = 1.06 * y.std() * n ** -0.2
    f = np.exp(-0.5 * ((g[:, None] - y[None]) / h) ** 2).mean(1) / (h * np.sqrt(2 * np.pi))
    score = np.gradient(np.log(f), g)
    tw = g + score
    # exact posterior mean for the two-component Gaussian prior
    def post_mean(yv):
        w = np.array([0.5 * stats.norm.pdf(yv, -2, np.sqrt(0.25 + 1)), 0.5 * stats.norm.pdf(yv, 1.5, np.sqrt(1 + 1))])
        m = np.array([(-2 / 0.25 + yv) / (1 / 0.25 + 1), (1.5 / 1.0 + yv) / (1 / 1.0 + 1)])
        return (w * m).sum(0) / w.sum(0)
    fig, ax = plt.subplots(figsize=(4.6, 3.0))
    ax.plot(g, g, color=GREY, ls=":", label=r"$y$ (no shrinkage)")
    ax.plot(g, post_mean(g), "k", label="exact $E[\\mu|y]$")
    ax.plot(g, tw, RED, ls="--", label="Tweedie with kernel score")
    ax.set_xlabel("y"); ax.set_ylabel(r"$E[\mu\mid y]$"); ax.set_xlim(-5, 5); ax.set_ylim(-4, 4)
    ax.legend(frameon=False, fontsize=8)
    fig.tight_layout()
    return _save(fig, "figS3-9", out_dir)


# ============================================================================= data export for the demonstrations
def export_json():
    os.makedirs(JSON_DIR, exist_ok=True)
    rng = np.random.default_rng(3001)
    x = np.sort(rng.uniform(-1, 1, 20))
    y = 0.3 + 1.2 * x - 1.5 * x ** 3 + rng.normal(0, 0.25, 20)
    d = {"schools": {"y": SCHOOLS_Y.tolist(), "sigma": SCHOOLS_S.tolist()},
         "rats": {"y": RATS_Y.astype(int).tolist(), "n": RATS_N.astype(int).tolist()},
         "poly": {"x": np.round(x, 4).tolist(), "y": np.round(y, 4).tolist(), "noise_sd": 0.25}}
    with open(os.path.join(JSON_DIR, "ch03-hier-data.json"), "w") as f:
        json.dump(d, f, separators=(",", ":"))
    print("wrote", os.path.join(JSON_DIR, "ch03-hier-data.json"))


FIGURES = {"3.1": fig_3_1, "3.2": fig_3_2, "3.3": fig_3_3, "3.4": fig_3_4, "3.5": fig_3_5, "3.6": fig_3_6,
           "3.7": fig_3_7, "3.8": fig_3_8, "3.10": fig_3_10, "3.11": fig_3_11, "3.12": fig_3_12, "3.13": fig_3_13,
           "3.15": fig_3_15, "3.16": fig_3_16, "3.17": fig_3_17, "3.18": fig_3_18, "3.20": fig_3_20,
           "3.22": fig_3_22, "3.23": fig_3_23, "3.25": fig_3_25, "3.27": fig_3_27, "3.28": fig_3_28, "3.29": fig_3_29,
           "S3.2": fig_S3_2, "S3.3": fig_S3_3, "S3.4": fig_S3_4, "S3.5": fig_S3_5, "S3.6": fig_S3_6,
           "S3.8": fig_S3_8, "S3.9": fig_S3_9}

if __name__ == "__main__":
    export_json()
    run(FIGURES, CH)
