"""Murphy (2023), Chapter 15: Generalized linear models (with Supplementary Chapter 15).

Companion code for Chapter 15 of K. P. Murphy, Probabilistic Machine Learning: Advanced Topics (MIT Press,
2023; December 2025 draft), pp. 591-630, and the supplement pp. 161-172, written from scratch with numpy,
scipy, PyTorch (only for automatic differentiation inside the sampler) and matplotlib, to be read next to
the lecture notes (../ch15-generalized-linear-models.html).

Methods implemented (the notes use these functions by name)
* GLM basics (15.1): exponential-family moments for the Gaussian, binomial and Poisson models, the zero-inflated
  Poisson (zip_pmf), the link functions, Newton / IRLS for a canonical GLM (glm_irls).
* Linear regression (15.2): the normal-inverse-gamma posterior (nig_posterior, (15.57)-(15.61)), its Student
  marginal and predictive (15.63), (15.66), the Jeffreys and g-prior special cases, Laplace approximation of a
  non-conjugate model (laplace_fit), the height-weight example on a simulated stand-in, spike-and-slab
  enumeration, the Bayesian lasso Gibbs sampler (blasso_gibbs), a horseshoe Gibbs sampler (horseshoe_gibbs),
  automatic relevancy determination by MacKay and by EM updates (ard_fit), and the matrix-normal
  inverse-Wishart update (mniw_posterior, (15.121)-(15.125)).
* Logistic regression (15.3): Newton's method for the MAP estimate and the Laplace approximation (logreg_map),
  Monte Carlo and probit approximations of the predictive ((15.147), (15.149)), a Hamiltonian Monte Carlo
  sampler with step size and mass adaptation (hmc), logit adjustment (15.140)-(15.142), the Berkeley
  admissions models, the Jaakkola-Jordan and Boehning variational posteriors (jj_vi, bohning_vi).
* Probit regression (15.4): MLE by BFGS and by EM (probit_em), the Albert-Chib Gibbs sampler (probit_gibbs).
* Multilevel GLMs (15.5): the radon-type hierarchical regression in centred and non-centred form, with HMC.
* Supplement: Jaakkola-Jordan and Boehning bounds (S15.1), beta-binomial regression (S15.2), the varying
  intercept and slope GLMM with an LKJ(2) prior (S15.3).

Data. No data files from the book are available. Stand-ins, said so in every caption: the Kalahari
height-weight data (Figures 15.1, 15.2) and the Minnesota radon data (Figures 15.15 to 15.17) are simulated
with the same structure (sizes, nonlinearity, 85 counties of 2 to 80 houses); the Berkeley admissions counts
are the real 12 cells of Table 15.1 (R's datasets::UCBAdmissions); Figure 15.8 uses the iris data of
scikit-learn; Figures 15.6, 15.7, 15.12 use simulated two-dimensional classification data. Seeds are fixed.

Figures. Reproduced here: 15.1, 15.2, 15.5, 15.6, 15.7, 15.8, 15.9, 15.11, 15.12, 15.15, 15.16, 15.17 and the
supplement's S15.1, S15.2, S15.3. Skipped, with the reason:
  15.3, 15.13, 15.14   graphical-model diagrams (nothing computed);
  15.4   a sketch of two Gaussian densities in the plane (the same calculation is made exact in 15.X2);
  15.10  causal DAGs (diagrams).
Extra figures for the gaps filled in the notes:
  15.X1  shrinkage profiles of the Gaussian, Laplace, horseshoe and spike-and-slab priors, and the horseshoe
         shrinkage coefficient;
  15.X2  automatic relevancy determination against ridge and the lasso on a sparse regression, and the
         marginal-likelihood geometry of Figure 15.4;
  15.X3  variational (Jaakkola-Jordan and Boehning) against Laplace and HMC posteriors of a logistic model;
  15.X4  the normal-normal partial pooling formula checked against the radon-type model.
Run `python3 ch15_generalized_linear_models.py --list`, `python3 ch15_generalized_linear_models.py` (about
ten minutes) or `--fig 15.8`. Files go to ../img/ch15/; the data used by the browser demonstrations go to
../data/ch15-*.json.
"""
import json
import os
import sys

os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")

import numpy as np
from scipy import optimize, special, stats

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import common  # noqa: E402
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, CYAN, plt, savefig, panel_label, sigmoid  # noqa: E402

CH = 15
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
WORK = os.path.join(HERE, "cache", "ch15-work")
os.makedirs(WORK, exist_ok=True)


def _save(fig, name, out_dir=None):
    return savefig(fig, CH, name, out_dir)


def _dump(name, obj):
    os.makedirs(DATA_OUT, exist_ok=True)
    p = os.path.join(DATA_OUT, "ch15-%s.json" % name)
    with open(p, "w") as f:
        json.dump(obj, f, separators=(",", ":"))
    return p


def log1pexp(a):
    return np.logaddexp(0.0, a)


# ============================================================================== GLM basics (15.1)
def zip_pmf(y, rho, lam):
    """Zero-inflated Poisson pmf, (15.24)."""
    y = np.asarray(y)
    p = (1 - rho) * stats.poisson.pmf(y, lam)
    return np.where(y == 0, rho + p, p)


def glm_irls(X, y, family="logistic", n_iter=50, tol=1e-12, ridge=0.0, N=None):
    """Newton / IRLS for a canonical GLM: family in {'logistic', 'poisson', 'gaussian'}.
    Returns (w, iterations). The Hessian of the NLL is X^T diag(f'(eta)) X, (15.38)."""
    n, d = X.shape
    w = np.zeros(d)
    Nn = np.ones(n) if N is None else np.asarray(N, float)
    for it in range(n_iter):
        eta = X @ w
        if family == "logistic":
            mu = sigmoid(eta); s = Nn * mu * (1 - mu); m = Nn * mu
        elif family == "poisson":
            mu = np.exp(eta); s = mu; m = mu
        else:
            m = eta; s = np.ones(n)
        g = X.T @ (m - y) + ridge * w
        H = X.T @ (s[:, None] * X) + ridge * np.eye(d)
        step = np.linalg.solve(H, g)
        w = w - step
        if np.max(np.abs(step)) < tol:
            break
    return w, it + 1


# ============================================================================== linear regression (15.2)
def nig_posterior(X, y, w0, V0inv, a0, b0):
    """Normal-inverse-gamma posterior (15.58)-(15.61) with prior w | s2 ~ N(w0, s2 V0), s2 ~ IG(a0, b0)."""
    N = len(y)
    Vninv = V0inv + X.T @ X
    Vn = np.linalg.inv(Vninv)
    wn = Vn @ (V0inv @ w0 + X.T @ y)
    an = a0 + N / 2
    bn = b0 + 0.5 * (w0 @ V0inv @ w0 + y @ y - wn @ Vninv @ wn)
    return wn, Vn, an, bn


def nig_predictive(Xt, wn, Vn, an, bn):
    """Student predictive (15.66): returns mean, scale matrix and dof (diag scale only)."""
    mean = Xt @ wn
    scale2 = (bn / an) * (1 + np.einsum("ij,jk,ik->i", Xt, Vn, Xt))
    return mean, np.sqrt(scale2), 2 * an


def logreg_nll_grad_hess(w, X, y, prior_prec):
    """Penalised NLL, gradient, Hessian of binary logistic regression with N(0, 1/prior_prec I) prior."""
    eta = X @ w
    mu = sigmoid(eta)
    nll = np.sum(log1pexp(eta) - y * eta) + 0.5 * prior_prec * w @ w
    g = X.T @ (mu - y) + prior_prec * w
    H = X.T @ ((mu * (1 - mu))[:, None] * X) + prior_prec * np.eye(len(w))
    return nll, g, H


def logreg_map(X, y, prior_var=100.0, n_iter=100, w0=None):
    """MAP by Newton's method for a spherical Gaussian prior; returns (w_map, H) with H the Hessian of the
    negative log posterior at the mode, so the Laplace approximation is N(w_map, H^{-1}), (15.145)."""
    d = X.shape[1]
    w = np.zeros(d) if w0 is None else w0.copy()
    lam = 1.0 / prior_var
    for _ in range(n_iter):
        _, g, H = logreg_nll_grad_hess(w, X, y, lam)
        step = np.linalg.solve(H, g)
        w = w - step
        if np.max(np.abs(step)) < 1e-12:
            break
    _, g, H = logreg_nll_grad_hess(w, X, y, lam)
    return w, H


def probit_predictive(mu, var):
    """Probit approximation (15.149): sigma(mu / sqrt(1 + pi var / 8))."""
    return sigmoid(mu / np.sqrt(1 + np.pi * var / 8))


# ============================================================================== Hamiltonian Monte Carlo
def hmc(logp, q0, n_samples=1000, n_warm=800, seed=0, delta=0.8, traj=1.5, max_leap=60, init_eps=0.05):
    """Hamiltonian Monte Carlo with leapfrog, a diagonal mass matrix and dual-averaging step size.

    logp(q) takes a float64 torch tensor and returns the log density (a scalar tensor); gradients come from
    autograd. Warm-up has three stages: step size, then step size while estimating the posterior variances
    (the inverse mass), then step size again. Returns a dict with samples, divergences, acceptance and eps.
    """
    import torch
    torch.set_num_threads(1)
    rng = np.random.default_rng(seed)
    q = np.array(q0, float)
    d = len(q)

    def vg(x):
        t = torch.tensor(x, dtype=torch.float64, requires_grad=True)
        lp = logp(t)
        g, = torch.autograd.grad(lp, t)
        return lp.item(), g.numpy().copy()

    inv_mass = np.ones(d)
    lp, g = vg(q)
    eps = init_eps
    mu_da = np.log(10 * eps)
    hbar, xbar, m_da = 0.0, 0.0, 0
    divs = 0
    draws, acc_all, dmask = [], [], []
    stage_a, stage_b = int(0.25 * n_warm), int(0.75 * n_warm)
    coll = []

    def reset_da(e):
        nonlocal mu_da, hbar, xbar, m_da
        mu_da, hbar, xbar, m_da = np.log(10 * e), 0.0, 0.0, 0

    for it in range(n_warm + n_samples):
        warm = it < n_warm
        p = rng.standard_normal(d) / np.sqrt(inv_mass)
        H0 = -lp + 0.5 * np.sum(inv_mass * p * p)
        e = eps * (1 + 0.1 * (rng.random() - 0.5) * 2)
        L = int(min(max_leap, max(1, round(traj / e))))
        qn, pn, lpn, gn = q.copy(), p.copy(), lp, g.copy()
        ok = True
        for _ in range(L):
            pn = pn + 0.5 * e * gn
            qn = qn + e * inv_mass * pn
            lpn, gn = vg(qn)
            if not np.isfinite(lpn):
                ok = False
                break
            pn = pn + 0.5 * e * gn
        if ok:
            H1 = -lpn + 0.5 * np.sum(inv_mass * pn * pn)
            dH = H1 - H0
            a = 1.0 if dH <= 0 else float(np.exp(-dH))
            isdiv = dH > 1000
        else:
            a = 0.0
            isdiv = True
        if not warm:
            divs += int(isdiv)
            dmask.append(bool(isdiv))
        if rng.random() < a:
            q, lp, g = qn, lpn, gn
        if warm:
            m_da += 1
            hbar = (1 - 1 / (m_da + 10)) * hbar + (delta - a) / (m_da + 10)
            x = mu_da - np.sqrt(m_da) / 0.05 * hbar
            eta = m_da ** (-0.75)
            xbar = eta * x + (1 - eta) * xbar
            eps = float(np.exp(x))
            if stage_a <= it < stage_b:
                coll.append(q.copy())
            if it == stage_b - 1:
                C = np.array(coll)
                v = C.var(0)
                n_c = len(C)
                inv_mass = (n_c / (n_c + 5)) * v + 1e-3 * (5 / (n_c + 5))
                inv_mass = np.maximum(inv_mass, 1e-8)
                reset_da(eps)
            if it == n_warm - 1:
                eps = float(np.exp(xbar))
        else:
            draws.append(q.copy())
            acc_all.append(a)
    return dict(samples=np.array(draws), div_mask=np.array(dmask), accept=float(np.mean(acc_all)), eps=eps, divergences=int(divs),
                inv_mass=inv_mass)


def ess_bulk(x):
    """Crude effective sample size of a 1-d chain from the initial positive autocorrelation sequence."""
    x = np.asarray(x, float) - np.mean(x)
    n = len(x)
    f = np.fft.rfft(x, 2 * n)
    ac = np.fft.irfft(f * np.conj(f))[:n]
    ac = ac / ac[0]
    s = 0.0
    for k in range(1, n - 1, 2):
        pair = ac[k] + ac[k + 1]
        if pair < 0:
            break
        s += pair
    return n / (1 + 2 * (s - 0.0))


# ============================================================================== Laplace approximation (autograd)
def laplace_fit(logp, q0, maxiter=500):
    """Mode and Hessian-based covariance of a torch log density: returns (mode, cov)."""
    import torch
    torch.set_num_threads(1)

    def f(x):
        t = torch.tensor(x, dtype=torch.float64, requires_grad=True)
        v = -logp(t)
        g, = torch.autograd.grad(v, t)
        return v.item(), g.numpy().copy()

    res = optimize.minimize(f, np.array(q0, float), jac=True, method="L-BFGS-B", options=dict(maxiter=maxiter, ftol=1e-14, gtol=1e-9))
    H = torch.autograd.functional.hessian(lambda t: -logp(t), torch.tensor(res.x, dtype=torch.float64)).numpy()
    return res.x, np.linalg.inv(H)


# ============================================================================== height and weight (Figures 15.1, 15.2)
def height_weight(seed=3):
    """A simulated stand-in for the Kalahari forager data: weight (kg), height (cm), adult flag.
    The mean curve is concave (children grow fast in weight and height, adults flatten)."""
    rng = np.random.default_rng(seed)
    n_ad, n_ch = 352, 190
    w_ad = rng.normal(45, 6.5, n_ad)
    w_ch = rng.uniform(4, 31, n_ch)
    w = np.concatenate([w_ad, w_ch])
    h = 58 + 3.2 * w - 0.0235 * w ** 2 + rng.normal(0, 5.0, len(w))
    adult = np.concatenate([np.ones(n_ad, bool), np.zeros(n_ch, bool)])
    return w, h, adult


def _hw_logpost(x, y, kinds, mean_alpha=178.0, sd_alpha=20.0, lam_sigma=None):
    """Return a torch log posterior in unconstrained coordinates (alpha, beta_1.., log sigma)."""
    import torch
    xt = torch.tensor(x, dtype=torch.float64)
    yt = torch.tensor(y, dtype=torch.float64)
    lam = 1.0 / np.std(y) if lam_sigma is None else lam_sigma
    K = len(kinds)

    def logp(q):
        alpha = q[0]
        betas, lj = [], 0.0
        lpri = -0.5 * ((alpha - mean_alpha) / sd_alpha) ** 2
        for k, kind in enumerate(kinds):
            z = q[1 + k]
            if kind == "ln":          # beta = exp(z), z ~ N(0,1): density of z is N(0,1) already
                betas.append(torch.exp(z)); lpri = lpri - 0.5 * z ** 2
            else:                      # beta ~ N(0, 1)
                betas.append(z); lpri = lpri - 0.5 * z ** 2
        ls = q[1 + K]
        sigma = torch.exp(ls)
        lpri = lpri + (-lam * sigma + ls)          # Exp(lam) prior with log-Jacobian
        mu = alpha + sum(b * (xt ** (j + 1)) for j, b in enumerate(betas))
        ll = -len(yt) * ls - 0.5 * torch.sum(((yt - mu) / sigma) ** 2)
        return ll + lpri
    return logp


def hw_posterior(x, y, kinds, n_post=2000, seed=0):
    """Laplace posterior for mu = alpha + sum_j beta_j x^j: returns draws of (alpha, betas, sigma)."""
    K = len(kinds)
    logp = _hw_logpost(x, y, kinds)
    q0 = np.zeros(2 + K); q0[0] = np.mean(y); q0[-1] = np.log(np.std(y))
    mode, cov = laplace_fit(logp, q0)
    rng = np.random.default_rng(seed)
    Z = rng.multivariate_normal(mode, cov, size=n_post)
    alpha = Z[:, 0]
    betas = np.stack([np.exp(Z[:, 1 + k]) if kinds[k] == "ln" else Z[:, 1 + k] for k in range(K)], 1)
    sigma = np.exp(Z[:, -1])
    return alpha, betas, sigma, mode


def fig_15_1(out=None):
    """Prior predictive with Gaussian and log-normal slope priors; posterior predictive (height-weight)."""
    w, h, adult = height_weight()
    wa, ha = w[adult], h[adult]
    xbar = wa.mean()
    rng = np.random.default_rng(1)
    xs = np.linspace(30, 65, 100)
    fig, axes = plt.subplots(1, 3, figsize=(11, 3.4))
    for ax, kind, ttl in ((axes[0], "n", r"$\beta\sim\mathcal{N}(0,10)$"), (axes[1], "ln", r"$\beta\sim\mathrm{LogNormal}(0,1)$")):
        for s in range(50):
            a = rng.normal(178, 20)
            b = rng.normal(0, 10) if kind == "n" else np.exp(rng.normal(0, 1))
            ax.plot(xs, a + b * (xs - xbar), color=BLUE, alpha=0.25, lw=1)
        ax.axhline(0, color="k", ls="--", lw=0.8); ax.axhline(272, color="k", ls="-", lw=1.0)
        ax.text(31, 280, "world's tallest man", fontsize=8)
        ax.set_xlim(30, 65); ax.set_ylim(-100, 450); ax.set_title(ttl)
        if kind == 'ln': ax.set_yticklabels([])
        ax.set_xlabel("weight (kg)")
    axes[0].set_ylabel("height (cm)")
    # posterior with the log-normal prior
    alpha, betas, sigma, mode = hw_posterior(wa - xbar, ha, ["ln"])
    mu = alpha[:, None] + betas[:, [0]] * (xs[None] - xbar)
    ypred = mu + sigma[:, None] * rng.standard_normal(mu.shape)
    ax = axes[2]
    ax.scatter(wa, ha, s=6, color=GREY, alpha=0.6)
    ax.fill_between(xs, *np.percentile(ypred, [2.5, 97.5], 0), color=BLUE, alpha=0.15, label="95% interval for $y$")
    ax.fill_between(xs, *np.percentile(mu, [2.5, 97.5], 0), color=BLUE, alpha=0.4, label=r"95% interval for $\mu$")
    ax.plot(xs, mu.mean(0), color=RED)
    ax.set_xlim(30, 65); ax.set_xlabel("weight (kg)"); ax.set_title("posterior predictive, log-normal prior"); ax.legend(loc="upper left", fontsize=8)
    ax.set_ylim(115, 195)
    print("   posterior mean of beta = %.3f, sigma = %.2f, alpha = %.1f" % (betas[:, 0].mean(), sigma.mean(), alpha.mean()))
    for a in axes:
        panel_label(a, "abc"[list(axes).index(a)], "lower right")
    _save(fig, "fig15-1", out)


def fig_15_2(out=None):
    """Polynomial regression of height on weight including children: three prior choices."""
    w, h, adult = height_weight()
    xs_ = (w - w.mean()) / w.std()
    grid = np.linspace(xs_.min(), xs_.max(), 120)
    cases = [(["ln"], "linear, log-normal $\\beta_1$"), (["ln", "ln"], "quadratic, log-normal $\\beta_2$"), (["ln", "n"], "quadratic, Gaussian $\\beta_2$")]
    rng = np.random.default_rng(2)
    fig, axes = plt.subplots(1, 3, figsize=(11, 3.4), sharey=True)
    out_vals = []
    for ax, (kinds, ttl) in zip(axes, cases):
        x_use = xs_
        if len(kinds) == 1:
            alpha, betas, sigma, _ = hw_posterior(x_use, h, kinds)
            mu = alpha[:, None] + betas[:, [0]] * grid[None]
        else:
            # polynomial features need powers: reuse the generic logp with x**j through hw_posterior on x
            alpha, betas, sigma, _ = hw_posterior(x_use, h, kinds)
            mu = alpha[:, None] + betas[:, [0]] * grid[None] + betas[:, [1]] * grid[None] ** 2
        yp = mu + sigma[:, None] * rng.standard_normal(mu.shape)
        ax.scatter(xs_, h, s=5, color=GREY, alpha=0.5)
        ax.fill_between(grid, *np.percentile(yp, [2.5, 97.5], 0), color=BLUE, alpha=0.15)
        ax.fill_between(grid, *np.percentile(mu, [2.5, 97.5], 0), color=BLUE, alpha=0.4)
        ax.plot(grid, mu.mean(0), color=RED)
        ax.set_title(ttl, fontsize=9.5); ax.set_xlabel("standardised weight")
        out_vals.append(betas.mean(0))
    axes[0].set_ylabel("height (cm)")
    for i, a in enumerate(axes):
        panel_label(a, "abc"[i], "lower right")
    print("   E[beta2|D]: log-normal prior %.3f, Gaussian prior %.3f" % (out_vals[1][1], out_vals[2][1]))
    _save(fig, "fig15-2", out)


# ============================================================================== priors for logistic regression (15.5)
def fig_15_5(out=None):
    """Prior on the logistic output under a N(0, omega) offset prior; fraction of 1s against dimension."""
    fig, axes = plt.subplots(1, 2, figsize=(9, 3.2))
    s = np.linspace(0.001, 0.999, 999)
    z = np.log(s / (1 - s))
    ax = axes[0]
    for om, col in ((10.0, RED), (1.5, BLUE)):
        dens = stats.norm.pdf(z, 0, np.sqrt(om)) / (s * (1 - s))
        ax.plot(s, dens, color=col, label="variance = %.1f" % om)
    ax.set_xlabel(r"$\sigma(\alpha)$"); ax.set_ylabel("density"); ax.set_ylim(0, 3.2); ax.legend(fontsize=8)
    ax.set_title(r"$\alpha\sim\mathcal{N}(0,\omega)$")
    ax = axes[1]
    rng = np.random.default_rng(0)
    S, N = 1500, 400
    bins = np.linspace(0, 1, 21)
    for D, col in ((1, BLUE), (5, GREEN), (15, RED)):
        fr = fraction_of_ones(rng, D, S, N, p=0.8, sd=1.5)
        c, _ = np.histogram(fr, bins, density=True)
        ax.step(0.5 * (bins[1:] + bins[:-1]), c, where="mid", color=col, label="D = %d" % D)
    ax.set_xlabel("fraction of labels equal to 1"); ax.set_title("prior std 1.5 on the weights"); ax.legend(fontsize=8)
    panel_label(axes[0], "a", "upper center"); panel_label(axes[1], "b", "upper center")
    fig.tight_layout()
    _save(fig, "fig15-5", out)


def fraction_of_ones(rng, D, S, N, p=0.8, sd=1.5):
    """Prior distribution of the fraction of ones among N labels from a logistic model with D binary features."""
    fr = np.empty(S)
    for s in range(S):
        X = (rng.random((N, D)) < p).astype(float)
        b = sd * rng.standard_normal(D)
        y = rng.random(N) < sigmoid(X @ b)
        fr[s] = y.mean()
    return fr


def fig_15_11(out=None):
    """The logistic function against the probit function Phi(lambda x), lambda^2 = pi/8."""
    x = np.linspace(-6, 6, 400)
    lam = np.sqrt(np.pi / 8)
    fig, ax = plt.subplots(figsize=(4.2, 3.2))
    ax.plot(x, sigmoid(x), color=RED, label=r"$\sigma(x)$")
    ax.plot(x, stats.norm.cdf(lam * x), color=BLUE, ls=":", lw=2.2, label=r"$\Phi(\lambda x)$, $\lambda=\sqrt{\pi/8}$")
    ax.legend(fontsize=8); ax.set_xlabel("x")
    print("   max |sigma(x) - Phi(lambda x)| = %.4f" % np.max(np.abs(sigmoid(x) - stats.norm.cdf(lam * x))))
    _save(fig, "fig15-11", out)


# ============================================================================== separable 2-d data (15.6, 15.7)
def separable_2d(seed=4, n=20, angle_deg=25.0, margin_deg=8.0):
    """Linearly separable two-class data in the plane by a line through the origin (no bias term)."""
    rng = np.random.default_rng(seed)
    wstar = np.array([np.cos(np.radians(angle_deg)), np.sin(np.radians(angle_deg))])
    X = []
    while len(X) < n:
        x = rng.normal(0, 2.2, 2)
        if abs(wstar @ x) / np.linalg.norm(x) > np.sin(np.radians(margin_deg)) and np.linalg.norm(x) > 1.0:
            X.append(x)
    X = np.array(X)
    y = (X @ wstar > 0).astype(float)
    return X, y


def loglik_grid(X, y, g1, g2):
    W1, W2 = np.meshgrid(g1, g2)
    eta = W1[..., None] * X[:, 0] + W2[..., None] * X[:, 1]
    return np.sum(y * eta - log1pexp(eta), -1), W1, W2


def fig_15_6(out=None):
    """Separable data, log likelihood, log unnormalised posterior and its Laplace approximation."""
    X, y = separable_2d()
    ws = np.array([[3, 1], [4, 2], [5, 3], [7, 3]], float)
    cols = [RED, GREEN, BLUE, MAGENTA]
    g = np.linspace(-2, 10, 241)
    ll, W1, W2 = loglik_grid(X, y, g, g)
    lp = ll - (W1 ** 2 + W2 ** 2) / 200
    wmap, H = logreg_map(X, y, 100.0)
    C = np.linalg.inv(H)
    d = np.stack([W1 - wmap[0], W2 - wmap[1]], -1)
    lap = -0.5 * np.einsum("...i,ij,...j->...", d, H, d)
    fig, axes = plt.subplots(1, 4, figsize=(13, 3.2))
    ax = axes[0]
    ax.scatter(X[y == 1, 0], X[y == 1, 1], marker="o", color=RED, s=16); ax.scatter(X[y == 0, 0], X[y == 0, 1], marker="s", color=BLUE, s=16)
    t = np.linspace(-6, 6, 10)
    for w, c in zip(ws, cols):
        ax.plot(t, -w[0] / w[1] * t, color=c, lw=1.4)
    ax.set_xlim(-6, 6); ax.set_ylim(-6, 6); ax.set_title("data and four boundaries"); ax.set_aspect("equal")
    for ax, Z, ttl, lv in ((axes[1], ll, "log likelihood", np.linspace(-14, 0, 15)), (axes[2], lp, "log unnormalised posterior", np.linspace(-14, 0, 15) + lp.max()), (axes[3], lap, "Laplace approximation", np.linspace(-6, 0, 13))):
        ax.contour(W1, W2, Z - (0 if Z is not lp else 0), levels=lv if Z is not lp else np.linspace(lp.max() - 14, lp.max(), 15), colors=GREY, linewidths=0.8)
        ax.set_title(ttl, fontsize=9.5); ax.set_xlabel("$w_1$"); ax.set_ylabel("$w_2$")
        ax.set_aspect("equal")
        for w, c in zip(ws, cols):
            ax.plot(*w, "o", color=c, ms=5)
    nrm = np.linalg.norm(ws, axis=1)
    wbig, _ = logreg_map(X, y, 1e6, n_iter=40)
    wbig = 10 * wbig / np.linalg.norm(wbig)
    axes[1].plot([0, wbig[0]], [0, wbig[1]], "k--", lw=0.8)
    axes[2].plot(*wmap, "x", color=RED, ms=8, mew=2); axes[3].plot(*wmap, "x", color=RED, ms=8, mew=2)
    for i, a in enumerate(axes):
        panel_label(a, "abcd"[i], "upper right")
    print("   weight norms of the four points: %s; MAP = %s" % (np.round(nrm, 2), np.round(wmap, 2)))
    fig.tight_layout()
    _save(fig, "fig15-6", out)
    _dump("logistic2d", dict(note="Simulated separable 2-d data for the Bayesian logistic demo (no bias term).",
                             x1=np.round(X[:, 0], 4).tolist(), x2=np.round(X[:, 1], 4).tolist(), y=y.astype(int).tolist(),
                             map_var100=np.round(wmap, 6).tolist(), H_var100=np.round(H, 6).tolist()))


def fig_15_7(out=None):
    """Posterior predictive for 2-d logistic regression: plug-in, sampled boundaries, MC average, probit."""
    X, y = separable_2d()
    wmap, H = logreg_map(X, y, 100.0)
    C = np.linalg.inv(H)
    rng = np.random.default_rng(5)
    Ws = rng.multivariate_normal(wmap, C, size=2000)
    g = np.linspace(-6, 6, 121)
    G1, G2 = np.meshgrid(g, g)
    P = np.stack([G1.ravel(), G2.ravel()], 1)
    p_plug = sigmoid(P @ wmap).reshape(G1.shape)
    p_mc = sigmoid(P @ Ws.T).mean(1).reshape(G1.shape)
    mu = P @ wmap
    var = np.einsum("ni,ij,nj->n", P, C, P)
    p_pro = probit_predictive(mu, var).reshape(G1.shape)
    fig, axes = plt.subplots(1, 4, figsize=(13, 3.3))
    for ax, Z, ttl in ((axes[0], p_plug, r"$p(y=1|x,\hat w_{\mathrm{map}})$"), (axes[2], p_mc, "MC average (2000 samples)"), (axes[3], p_pro, "probit approximation")):
        cs = ax.contour(G1, G2, Z, levels=np.linspace(0.1, 0.9, 9), colors=[BLUE] * 9, linewidths=0.9)
        ax.set_title(ttl, fontsize=9.5)
    t = np.array([-6, 6.0])
    for k in range(20):
        axes[1].plot(t, -Ws[k, 0] / Ws[k, 1] * t, color=BLUE, alpha=0.5, lw=1)
    axes[1].set_title("decision boundaries of 20 samples", fontsize=9.5)
    for ax in axes:
        ax.scatter(X[y == 1, 0], X[y == 1, 1], marker="o", color=RED, s=14, zorder=3); ax.scatter(X[y == 0, 0], X[y == 0, 1], marker="s", color=GREEN, s=14, zorder=3)
        ax.set_xlim(-6, 6); ax.set_ylim(-6, 6); ax.set_aspect("equal")
    for i, a in enumerate(axes):
        panel_label(a, "abcd"[i], "lower right")
    print("   max |MC - probit| = %.4f, mean = %.4f" % (np.max(np.abs(p_mc - p_pro)), np.mean(np.abs(p_mc - p_pro))))
    fig.tight_layout()
    _save(fig, "fig15-7", out)


# ============================================================================== iris, HMC (15.8)
def iris_setosa_versicolor(n0, n1, seed=0):
    from sklearn.datasets import load_iris
    d = load_iris()
    X = d.data[:, :2]
    yy = d.target
    rng = np.random.default_rng(seed)
    i0 = rng.choice(np.where(yy == 0)[0], n0, replace=False)
    i1 = rng.choice(np.where(yy == 1)[0], n1, replace=False)
    idx = np.concatenate([i0, i1])
    return X[idx], np.concatenate([np.zeros(n0), np.ones(n1)])


def logreg_hmc(X, y, prior_sd=10.0, n=2000, seed=0):
    """HMC for logistic regression with intercept; the columns of X are centred inside, the draws returned are
    (b, w1, w2) for the original coordinates."""
    import torch
    m = X.mean(0)
    Xc = torch.tensor(X - m, dtype=torch.float64)
    yt = torch.tensor(y, dtype=torch.float64)

    def logp(q):
        eta = q[0] + Xc @ q[1:]
        return torch.sum(yt * eta - torch.nn.functional.softplus(eta)) - 0.5 * torch.sum(q ** 2) / prior_sd ** 2
    r = hmc(logp, np.zeros(1 + X.shape[1]), n_samples=n, n_warm=600, seed=seed)
    Q = r["samples"].copy()
    Q[:, 0] = Q[:, 0] - Q[:, 1:] @ m
    r["samples"] = Q
    return r


def fig_15_8(out=None):
    """Posterior over the decision boundary for setosa against versicolor (sepal length and width), HMC."""
    fig, axes = plt.subplots(1, 2, figsize=(9.4, 3.6), sharey=True)
    for ax, (n0, n1), lab in ((axes[0], (25, 25), "a"), (axes[1], (5, 45), "b")):
        X, y = iris_setosa_versicolor(n0, n1, seed=1)
        r = logreg_hmc(X, y, 10.0, 2000, seed=2)
        Q = r["samples"]
        xs = np.linspace(X[:, 0].min() - 0.3, X[:, 0].max() + 0.3, 100)
        B = -(Q[:, [0]] + Q[:, [1]] * xs[None]) / Q[:, [2]]
        ax.fill_between(xs, *np.percentile(B, [2.5, 97.5], 0), color=GREY, alpha=0.35)
        ax.plot(xs, B.mean(0), color="k", lw=1.5)
        ax.scatter(X[y == 0, 0], X[y == 0, 1], color=RED, s=14, label="setosa")
        ax.scatter(X[y == 1, 0], X[y == 1, 1], color=BLUE, marker="s", s=14, label="versicolor")
        ax.set_xlabel("sepal length"); ax.set_ylim(1.8, 4.6)
        ax.set_title("%d + %d examples" % (n0, n1), fontsize=10)
        ax.legend(loc="upper right", fontsize=8)
        panel_label(ax, lab, "lower left")
        print("   %s: accept %.2f, eps %.3f, divergences %d, boundary width at mean x: %.3f" % (lab, r["accept"], r["eps"], r["divergences"], np.diff(np.percentile(B, [2.5, 97.5], 0)[:, 50])[0]))
    axes[0].set_ylabel("sepal width")
    fig.tight_layout()
    _save(fig, "fig15-8", out)


# ============================================================================== Berkeley (15.3.9)
def berkeley():
    """The 12 cells of Table 15.1: arrays admitted, applicants, department index 0..5, male indicator."""
    A = np.array([512, 89, 353, 17, 120, 202, 138, 131, 53, 94, 22, 24], float)
    R = np.array([313, 19, 207, 8, 205, 391, 279, 244, 138, 299, 351, 317], float)
    dept = np.repeat(np.arange(6), 2)
    male = np.tile([1, 0], 6)
    return A, A + R, dept, male


def berkeley_hmc(model, n=4000, seed=0):
    """Models of Section 15.3.9: 'gender' (15.157-15.159), 'dept' (15.160-15.163), 'betabin' (S15.64-S15.68)."""
    import torch
    A, N, dept, male = berkeley()
    At, Nt = torch.tensor(A), torch.tensor(N)
    g = torch.tensor(1 - male)                 # 0 = male, 1 = female  (index into alpha)
    dp = torch.tensor(dept)
    if model == "gender":
        def logp(q):
            eta = q[g]
            return torch.sum(At * eta - Nt * torch.nn.functional.softplus(eta)) - 0.5 * torch.sum(q ** 2) / 1.5 ** 2
        q0 = np.zeros(2)
    elif model == "dept":
        def logp(q):
            eta = q[g] + q[2 + dp]
            return torch.sum(At * eta - Nt * torch.nn.functional.softplus(eta)) - 0.5 * torch.sum(q ** 2) / 1.5 ** 2
        q0 = np.zeros(8)
    else:
        def logp(q):                           # q = (alpha_male, alpha_female, log phi); kappa = phi + 2
            phi = torch.exp(q[2])
            kappa = phi + 2
            pi = torch.sigmoid(q[g])
            a, b = pi * kappa, (1 - pi) * kappa
            ll = torch.lgamma(Nt + 1) - torch.lgamma(At + 1) - torch.lgamma(Nt - At + 1) + torch.lgamma(At + a) + torch.lgamma(Nt - At + b) - torch.lgamma(Nt + a + b) + torch.lgamma(a + b) - torch.lgamma(a) - torch.lgamma(b)
            return torch.sum(ll) - 0.5 * torch.sum(q[:2] ** 2) / 1.5 ** 2 - phi + q[2]
        q0 = np.zeros(3)
    return hmc(logp, q0, n_samples=n, n_warm=800, seed=seed)


def _berk_panel(ax, mu, Apred, title):
    A, N, dept, male = berkeley()
    pos = np.arange(12)
    ax.scatter(pos, A / N, color=BLUE, s=22, zorder=3)
    ax.errorbar(pos, mu.mean(0), yerr=mu.std(0), fmt="o", mfc="none", mec="k", ecolor="k", ms=5, lw=0.9, zorder=4)
    lo, hi = np.percentile(Apred / N, [2.5, 97.5], 0)
    ax.plot(pos, lo, "k+", ms=7); ax.plot(pos, hi, "k+", ms=7)
    ax.set_xticks(pos[::2] + 0.5); ax.set_xticklabels(list("ABCDEF"))
    ax.set_xlabel("department (male left, female right)"); ax.set_ylim(0, 1); ax.set_title(title, fontsize=9.5)
    for k in range(0, 12, 2):
        ax.axvline(k - 0.5, color="#dddddd", lw=0.6)


def fig_15_9(out=None):
    """Berkeley admissions: posterior predictive of the gender-only and the gender-plus-department model."""
    A, N, dept, male = berkeley()
    rng = np.random.default_rng(0)
    fig, axes = plt.subplots(1, 2, figsize=(10, 3.4), sharey=True)
    summ = {}
    for ax, model, ttl in ((axes[0], "gender", "gender only"), (axes[1], "dept", "gender and department")):
        r = berkeley_hmc(model, 4000, 1)
        Q = r["samples"]
        g = 1 - male
        eta = Q[:, g] + (Q[:, 2 + dept] if model == "dept" else 0)
        mu = sigmoid(eta)
        Apred = rng.binomial(N.astype(int)[None], mu)
        _berk_panel(ax, mu, Apred, ttl)
        a1, a2 = Q[:, 0], Q[:, 1]       # male, female
        diff = sigmoid(a1) - sigmoid(a2)
        summ[model] = dict(a_male=np.percentile(a1, [5.5, 94.5]), a_female=np.percentile(a2, [5.5, 94.5]), diff=np.percentile(diff, [5.5, 94.5]), diff_mean=diff.mean(), eps=r["eps"], acc=r["accept"], div=r["divergences"], rhat_ess=ess_bulk(a1))
        print("   %-6s: 89%% CI alpha_male %s, alpha_female %s, difference %s, mean %.3f (ess %.0f, div %d)" % (model, np.round(summ[model]["a_male"], 2), np.round(summ[model]["a_female"], 2), np.round(summ[model]["diff"], 2), diff.mean(), summ[model]["rhat_ess"], r["divergences"]))
    axes[0].set_ylabel("admission rate")
    for i, a in enumerate(axes):
        panel_label(a, "ab"[i], "upper right")
    fig.tight_layout()
    _save(fig, "fig15-9", out)


# ============================================================================== probit (15.4)
def probit_nll(w, X, y, lam):
    eta = X @ w
    return -np.sum(y * special.log_ndtr(eta) + (1 - y) * special.log_ndtr(-eta)) + 0.5 * lam * w @ w


def probit_grad(w, X, y, lam):
    """Gradient of the penalised NLL; with ytilde = 2y - 1 the per-example term is (15.170)."""
    eta = X @ w
    yt = 2 * y - 1
    r = yt * stats.norm.pdf(eta) / np.exp(special.log_ndtr(yt * eta))
    return -X.T @ r + lam * w


def probit_hess(w, X, y, lam):
    """Hessian of the penalised NLL, from (15.171)."""
    eta = X @ w
    yt = 2 * y - 1
    ph = stats.norm.pdf(eta)
    Ph = np.exp(special.log_ndtr(yt * eta))
    s = ph ** 2 / Ph ** 2 + yt * eta * ph / Ph
    return X.T @ (s[:, None] * X) + lam * np.eye(len(w))


def probit_em(X, y, lam=0.01, n_iter=200, w0=None, record=False):
    """EM for probit regression with a N(0, I/lam) prior: E step (15.175), M step (15.176)."""
    d = X.shape[1]
    w = np.zeros(d) if w0 is None else w0.copy()
    A = np.linalg.inv(lam * np.eye(d) + X.T @ X)
    hist = [probit_nll(w, X, y, lam)]
    for _ in range(n_iter):
        mu = X @ w
        ez = np.where(y == 1, mu + stats.norm.pdf(mu) / np.exp(special.log_ndtr(mu)), mu - stats.norm.pdf(mu) / np.exp(special.log_ndtr(-mu)))
        w = A @ (X.T @ ez)
        hist.append(probit_nll(w, X, y, lam))
    return (w, np.array(hist)) if record else w


def probit_gibbs(X, y, V0inv, n=2000, seed=0):
    """Albert-Chib Gibbs sampler (15.177)-(15.180) with a N(0, V0) prior."""
    rng = np.random.default_rng(seed)
    N_, d = X.shape
    Vn = np.linalg.inv(V0inv + X.T @ X)
    L = np.linalg.cholesky(Vn)
    w = np.zeros(d)
    out = np.empty((n, d))
    for t in range(n):
        mu = X @ w
        u = rng.random(N_)
        lo = np.where(y == 1, stats.norm.cdf(-mu), 0.0)       # z > 0 -> Phi(z - mu) in (Phi(-mu), 1)
        hi = np.where(y == 1, 1.0, stats.norm.cdf(-mu))
        z = mu + stats.norm.ppf(np.clip(lo + u * (hi - lo), 1e-300, 1 - 1e-16))
        w = Vn @ (X.T @ z) + L @ rng.standard_normal(d)
        out[t] = w
    return out


def probit_data(seed=7, n=150):
    rng = np.random.default_rng(seed)
    X0 = rng.multivariate_normal([-1.2, -0.5], [[1.2, 0.4], [0.4, 0.9]], n // 2)
    X1 = rng.multivariate_normal([1.0, 0.9], [[1.0, -0.3], [-0.3, 1.1]], n // 2)
    X = np.vstack([X0, X1])
    y = np.r_[np.zeros(n // 2), np.ones(n // 2)]
    return np.hstack([np.ones((n, 1)), X]), y


def fig_15_12(out=None):
    """Probit regression in 2-d: BFGS against EM, penalised negative log likelihood against iterations."""
    X, y = probit_data()
    lam = 0.01
    hist_b = []
    res = optimize.minimize(probit_nll, np.zeros(3), args=(X, y, lam), jac=probit_grad, method="BFGS", callback=lambda w: hist_b.append(probit_nll(w, X, y, lam)), options=dict(gtol=1e-10))
    best = res.fun
    w_em, hist_em = probit_em(X, y, lam, 400, record=True)
    fig, axes = plt.subplots(1, 2, figsize=(9.4, 3.4))
    ax = axes[0]
    ax.scatter(X[y == 0, 1], X[y == 0, 2], color=RED, s=12); ax.scatter(X[y == 1, 1], X[y == 1, 2], color=BLUE, marker="s", s=12)
    t = np.linspace(-4, 4, 5)
    ax.plot(t, -(res.x[0] + res.x[1] * t) / res.x[2], color="k")
    ax.set_xlabel("$x_1$"); ax.set_ylabel("$x_2$"); ax.set_title("probit fit, L2 regulariser %.3f" % lam, fontsize=9.5)
    ax = axes[1]
    ax.semilogy(np.arange(len(hist_b)) + 1, np.maximum(np.array(hist_b) - best, 1e-13), color=RED, label="BFGS (quasi-Newton)")
    ax.semilogy(np.arange(len(hist_em)), np.maximum(hist_em - best, 1e-13), color=BLUE, label="EM")
    ax.set_xlabel("iteration"); ax.set_ylabel("penalised NLL minus optimum"); ax.legend(fontsize=8); ax.set_xlim(0, 150)
    for i, a in enumerate(axes):
        panel_label(a, "ab"[i], "upper right")
    n_em = int(np.argmax(hist_em - best < 1e-8))
    print("   BFGS iterations %d, EM iterations to 1e-8: %d, |w_em - w_bfgs| = %.2e" % (len(hist_b), n_em, np.max(np.abs(w_em - res.x))))
    fig.tight_layout()
    _save(fig, "fig15-12", out)


# ============================================================================== radon-type hierarchical regression (15.5)
def radon_standin(seed=11, J=85):
    """Simulated stand-in for the Minnesota radon data: J counties with 2 to 80 houses each, a binary floor
    indicator (1 = first floor, about 20% of houses), and county intercepts and slopes drawn from a Gaussian."""
    rng = np.random.default_rng(seed)
    nj = np.clip(np.round(np.exp(rng.normal(2.4, 1.0, J))).astype(int) + 2, 2, 80)
    county = np.repeat(np.arange(J), nj)
    floor = (rng.random(len(county)) < 0.2).astype(float)
    mu_a, mu_b, sa, sb, sy = 1.5, -0.6, 0.35, 0.2, 0.75
    a = rng.normal(mu_a, sa, J)
    b = rng.normal(mu_b, sb, J)
    y = a[county] + b[county] * floor + rng.normal(0, sy, len(county))
    return dict(y=y, floor=floor, county=county, J=J, nj=nj, true=dict(a=a, b=b, mu_a=mu_a, mu_b=mu_b, sa=sa, sb=sb, sy=sy))


def _hc_logpdf(s):
    """log of the half-Cauchy(1) density on s > 0 plus the log-Jacobian of s = exp(l): returns the density of l."""
    import torch
    return torch.log(torch.tensor(2 / np.pi)) - torch.log1p(s ** 2) + torch.log(s)


def radon_logp(D, centered):
    import torch
    y = torch.tensor(D["y"]); fl = torch.tensor(D["floor"]); cn = torch.tensor(D["county"]); J = D["J"]

    def logp(q):
        mua, mub = q[0], q[1]
        sa, sb, sy = torch.exp(q[2]), torch.exp(q[3]), torch.exp(q[4])
        u, v = q[5:5 + J], q[5 + J:5 + 2 * J]
        if centered:
            a, b = u, v
            lp = -0.5 * torch.sum(((a - mua) / sa) ** 2) - J * q[2] - 0.5 * torch.sum(((b - mub) / sb) ** 2) - J * q[3]
        else:
            a, b = mua + sa * u, mub + sb * v
            lp = -0.5 * torch.sum(u ** 2) - 0.5 * torch.sum(v ** 2)
        mu = a[cn] + b[cn] * fl
        ll = -len(y) * q[4] - 0.5 * torch.sum(((y - mu) / sy) ** 2)
        pri = -0.5 * mua ** 2 - 0.5 * mub ** 2 + _hc_logpdf(sa) + _hc_logpdf(sb) + _hc_logpdf(sy)
        return ll + lp + pri
    return logp


def radon_fit(centered, n=1000, n_warm=700, seed=0, force=False):
    """HMC for the hierarchical radon model; cached on disk. Returns dict with arrays mua, mub, sa, sb, sy, a, b."""
    D = radon_standin()
    tag = os.path.join(WORK, "radon_%s.npz" % ("c" if centered else "nc"))
    J = D["J"]
    if os.path.exists(tag) and not force:
        z = np.load(tag)
        return {k: z[k] for k in z.files}, D
    q0 = np.zeros(5 + 2 * J)
    q0[0] = D["y"].mean(); q0[2:5] = np.log([0.3, 0.2, 0.7])
    if centered:
        q0[5:5 + J] = D["y"].mean()
    r = hmc(radon_logp(D, centered), q0, n_samples=n, n_warm=n_warm, seed=seed, delta=0.9, traj=2.0, max_leap=80)
    Q = r["samples"]
    out = dict(mua=Q[:, 0], mub=Q[:, 1], sa=np.exp(Q[:, 2]), sb=np.exp(Q[:, 3]), sy=np.exp(Q[:, 4]), div_mask=r["div_mask"], eps=np.array(r["eps"]), accept=np.array(r["accept"]))
    if centered:
        out["a"], out["b"] = Q[:, 5:5 + J], Q[:, 5 + J:]
        out["ut"], out["vt"] = (Q[:, 5:5 + J] - out["mua"][:, None]) / out["sa"][:, None], (Q[:, 5 + J:] - out["mub"][:, None]) / out["sb"][:, None]
    else:
        out["ut"], out["vt"] = Q[:, 5:5 + J], Q[:, 5 + J:]
        out["a"] = out["mua"][:, None] + out["sa"][:, None] * Q[:, 5:5 + J]
        out["b"] = out["mub"][:, None] + out["sb"][:, None] * Q[:, 5 + J:]
    np.savez(tag, **out)
    return out, D


def radon_nonhier(D, prior_sd=10.0, n=400, seed=0):
    """Independent (unpooled) Bayesian regression per county with a weak N(0, prior_sd^2) prior, known noise sd
    equal to the residual sd of the pooled least-squares fit. Returns draws of shape (n, J, 2) and the sd."""
    rng = np.random.default_rng(seed)
    J = D["J"]
    X = np.stack([np.ones(len(D["y"])), D["floor"]], 1)
    wpool = np.linalg.lstsq(X, D["y"], rcond=None)[0]
    sy = np.std(D["y"] - X @ wpool)
    out = np.empty((n, J, 2)); means = np.empty((J, 2))
    for j in range(J):
        m = D["county"] == j
        Xj, yj = X[m], D["y"][m]
        P = Xj.T @ Xj / sy ** 2 + np.eye(2) / prior_sd ** 2
        C = np.linalg.inv(P)
        mu = C @ Xj.T @ yj / sy ** 2
        out[:, j] = rng.multivariate_normal(mu, C, n)
        means[j] = mu
    return out, means, sy


def fig_15_15(out=None):
    """Posterior intervals of the county intercepts and slopes in the hierarchical radon model."""
    R, D = radon_fit(False)
    fig, axes = plt.subplots(1, 2, figsize=(9.2, 4.6))
    for ax, key, mkey, lab in ((axes[0], "a", "mua", r"intercept $\alpha_j$"), (axes[1], "b", "mub", r"slope $\beta_j$")):
        M = R[key]
        mean = M.mean(0)
        order = np.argsort(mean)
        lo, hi = np.percentile(M, [3, 97], 0)
        y = np.arange(D["J"])
        ax.hlines(y, lo[order], hi[order], color=BLUE, lw=1.0)
        ax.plot(mean[order], y, "o", color="k", ms=2.5)
        ax.axvline(R[mkey].mean(), color=RED, lw=1.0, ls="--")
        ax.set_xlabel(lab); ax.set_ylabel("county (sorted)")
        ax.set_yticks([])
    for i, a in enumerate(axes):
        panel_label(a, "ab"[i], "upper left")
    print("   posterior means: mu_alpha %.2f, mu_beta %.2f, sigma_alpha %.2f, sigma_beta %.2f, sigma_y %.2f; divergences %d" % (R["mua"].mean(), R["mub"].mean(), R["sa"].mean(), R["sb"].mean(), R["sy"].mean(), R["div_mask"].sum()))
    fig.tight_layout()
    _save(fig, "fig15-15", out)


def radon_example_counties(D):
    nj, fl, cn = D["nj"], D["floor"], D["county"]
    n1 = np.array([fl[cn == j].sum() for j in range(D["J"])])
    A = int(np.where((n1 == 0) & (nj >= 3))[0][np.argmax(nj[(n1 == 0) & (nj >= 3)])]) if np.any((n1 == 0) & (nj >= 3)) else int(np.argmin(nj))
    B = int(np.argmin(np.abs(nj - 15) + 100 * (n1 < 2)))
    C = int(np.argmax(nj))
    return [A, B, C]


def fig_15_16(out=None):
    """Predictions from the hierarchical (red) and non-hierarchical (blue) radon models for three counties."""
    R, D = radon_fit(False)
    NH, NHm, sy = radon_nonhier(D)
    js = radon_example_counties(D)
    rng = np.random.default_rng(3)
    fig, axes = plt.subplots(1, 3, figsize=(10.4, 3.3), sharey=True)
    for ax, j in zip(axes, js):
        m = D["county"] == j
        xj = D["floor"][m] + rng.uniform(-0.04, 0.04, m.sum())
        ax.scatter(xj, D["y"][m], color="k", s=14, zorder=4)
        for s_ in range(20):
            k = rng.integers(len(R["mua"]))
            ax.plot([0, 1], [R["a"][k, j], R["a"][k, j] + R["b"][k, j]], color=RED, alpha=0.2, lw=0.8)
            ax.plot([0, 1], [NH[s_, j, 0], NH[s_, j, 0] + NH[s_, j, 1]], color=BLUE, alpha=0.2, lw=0.8)
        ax.plot([0, 1], [R["a"][:, j].mean(), R["a"][:, j].mean() + R["b"][:, j].mean()], color=RED, lw=2.2, label="hierarchical")
        ax.plot([0, 1], [NHm[j, 0], NHm[j, 0] + NHm[j, 1]], color=BLUE, lw=2.2, label="non-hierarchical")
        n1 = int(D["floor"][m].sum())
        ax.set_title("county %d: %d houses, %d on the first floor" % (j + 1, m.sum(), n1), fontsize=9)
        ax.set_xticks([0, 1]); ax.set_xticklabels(["basement", "first floor"]); ax.set_xlim(-0.3, 1.3)
        ax.set_ylim(-1.5, 5)
    axes[0].set_ylabel("log radon level"); axes[0].legend(fontsize=8, loc="lower left")
    for i, a in enumerate(axes):
        panel_label(a, "abc"[i], "upper right")
    # accuracy against the simulated truth
    T = D["true"]
    tm = np.concatenate([T["a"], T["a"] + T["b"]]); hm = np.concatenate([R["a"].mean(0), R["a"].mean(0) + R["b"].mean(0)]); nm = np.concatenate([NHm[:, 0], NHm.sum(1)])
    print("   RMSE to the true county lines: hierarchical %.3f, non-hierarchical %.3f" % (np.sqrt(np.mean((hm - tm) ** 2)), np.sqrt(np.mean((nm - tm) ** 2))))
    X = np.stack([np.ones(len(D["y"])), D["floor"]], 1)
    fit_h = R["a"].mean(0)[D["county"]] + R["b"].mean(0)[D["county"]] * D["floor"]
    fit_n = NHm[D["county"], 0] + NHm[D["county"], 1] * D["floor"]
    print("   in-sample RMSE: hierarchical %.3f, non-hierarchical %.3f" % (np.sqrt(np.mean((D["y"] - fit_h) ** 2)), np.sqrt(np.mean((D["y"] - fit_n) ** 2))))
    fig.tight_layout()
    _save(fig, "fig15-16", out)


def fig_15_17(out=None):
    """The funnel: p(beta_j, sigma_beta | D) in the centred and the non-centred parameterisation."""
    Rc, D = radon_fit(True)
    Rn, _ = radon_fit(False)
    j = int(np.argmin(D["nj"]))
    fig, axes = plt.subplots(1, 2, figsize=(9.2, 3.8))
    for ax, R, key, lab in ((axes[0], Rc, "b", r"slope $\beta_j$ (centred)"), (axes[1], Rn, "vt", r"offset $\tilde\beta_j$ (non-centred)")):
        x = R[key][:, j]
        dm = R["div_mask"]
        ax.scatter(x[~dm], R["sb"][~dm], s=4, color=BLUE, alpha=0.4)
        ax.scatter(x[dm], R["sb"][dm], s=8, color=RED, label="divergent (%d)" % dm.sum())
        ax.set_xlabel(lab); ax.set_ylabel(r"$\sigma_\beta$"); ax.set_ylim(0, None)
        ax.set_title("county %d (%d houses); min $\\sigma_\\beta$ = %.3f" % (j + 1, D["nj"][j], R["sb"].min()), fontsize=9.5)
        if dm.sum():
            ax.legend(fontsize=8)
    for i, a in enumerate(axes):
        panel_label(a, "ab"[i], "upper right")
    print("   centred: divergences %d, eps %.3f, min sigma_b %.4f, ESS(sigma_b) %.0f;  non-centred: divergences %d, eps %.3f, ESS %.0f" % (Rc["div_mask"].sum(), Rc["eps"], Rc["sb"].min(), ess_bulk(Rc["sb"]), Rn["div_mask"].sum(), Rn["eps"], ess_bulk(Rn["sb"])))
    fig.tight_layout()
    _save(fig, "fig15-17", out)


# ============================================================================== variational bounds (supplement 15.1)
def jj_lambda(psi):
    """lambda(psi) = tanh(psi/2) / (4 psi) = (sigma(psi) - 1/2) / (2 psi), (S15.5); equals 1/8 at psi = 0."""
    psi = np.asarray(psi, float)
    small = np.abs(psi) < 1e-8
    safe = np.where(small, 1.0, psi)
    return np.where(small, 0.125, np.tanh(safe / 2) / (4 * safe))


def jj_bound(eta, psi):
    """Jaakkola-Jordan lower bound on sigma(eta), (S15.4)."""
    return sigmoid(psi) * np.exp((eta - psi) / 2 - jj_lambda(psi) * (eta ** 2 - psi ** 2))


def bohning_logistic_bound(eta, psi):
    """Boehning quadratic upper bound on log(1 + e^eta), (S15.42)-(S15.45): a = 1/4."""
    a = 0.25
    b = a * psi - sigmoid(psi)
    c = 0.5 * a * psi ** 2 - sigmoid(psi) * psi + log1pexp(psi)
    return 0.5 * a * eta ** 2 - b * eta + c


def kl_gauss(m, V, m0, V0):
    d = len(m)
    V0i = np.linalg.inv(V0)
    return 0.5 * (np.trace(V0i @ V) + (m - m0) @ V0i @ (m - m0) - d + np.linalg.slogdet(V0)[1] - np.linalg.slogdet(V)[1])


def jj_vi(X, y, m0, V0, n_iter=50):
    """Jaakkola-Jordan variational posterior, (S15.9)-(S15.11), (S15.15); returns (mu, V, psi, elbo history)."""
    N = len(y)
    V0i = np.linalg.inv(V0)
    psi = np.ones(N)
    hist = []
    for _ in range(n_iter):
        lam = jj_lambda(psi)
        Vi = V0i + 2 * (X.T * lam) @ X
        V = np.linalg.inv(Vi)
        mu = V @ (V0i @ m0 + X.T @ (y - 0.5))
        hist.append(0.5 * (np.linalg.slogdet(V)[1] - np.linalg.slogdet(V0)[1]) + 0.5 * mu @ Vi @ mu - 0.5 * m0 @ V0i @ m0
                    + np.sum(np.log(sigmoid(psi)) - psi / 2 + lam * psi ** 2))
        psi = np.sqrt(np.einsum("ni,ij,nj->n", X, V + np.outer(mu, mu), X))
    return mu, V, psi, np.array(hist)


def bohning_vi(X, y, m0, V0, n_iter=50):
    """Boehning (constant curvature) variational posterior for binary logistic regression; the ELBO history
    uses the bound itself, so it is a lower bound on the log evidence."""
    N, d = X.shape
    a = 0.25
    V0i = np.linalg.inv(V0)
    V = np.linalg.inv(V0i + a * X.T @ X)
    psi = np.zeros(N)
    hist = []
    for _ in range(n_iter):
        b = a * psi - sigmoid(psi)
        mu = V @ (V0i @ m0 + X.T @ (y + b))
        m = X @ mu
        v = np.einsum("ni,ij,nj->n", X, V, X)
        c = 0.5 * a * psi ** 2 - sigmoid(psi) * psi + log1pexp(psi)
        elbo = -kl_gauss(mu, V, m0, V0) + np.sum(y * m - (0.5 * a * (m ** 2 + v) - b * m + c))
        hist.append(elbo)
        psi = m
    return mu, V, psi, np.array(hist)


def exact_logistic_posterior_2d(X, y, prior_sd, lim=8.0, n=401):
    """Grid quadrature of the posterior of a no-bias logistic model in 2-d: returns log evidence, mean, covariance, grid."""
    g = np.linspace(-lim, lim, n)
    ll, W1, W2 = loglik_grid(X, y, g, g)
    lp = ll - (W1 ** 2 + W2 ** 2) / (2 * prior_sd ** 2) - np.log(2 * np.pi * prior_sd ** 2)
    dA = (g[1] - g[0]) ** 2
    mx = lp.max()
    Z = np.exp(lp - mx)
    ev = np.log(Z.sum() * dA) + mx
    P = Z / Z.sum()
    m = np.array([np.sum(P * W1), np.sum(P * W2)])
    C = np.array([[np.sum(P * (W1 - m[0]) ** 2), np.sum(P * (W1 - m[0]) * (W2 - m[1]))], [0, np.sum(P * (W2 - m[1]) ** 2)]])
    C[1, 0] = C[0, 1]
    return ev, m, C, (W1, W2, P)


def laplace_log_evidence(X, y, prior_var):
    """Laplace estimate of log p(D): log joint at the mode + (d/2) log 2 pi - (1/2) log det H."""
    w, H = logreg_map(X, y, prior_var)
    d = len(w)
    eta = X @ w
    lj = np.sum(y * eta - log1pexp(eta)) - 0.5 * w @ w / prior_var - 0.5 * d * np.log(2 * np.pi * prior_var)
    return lj + 0.5 * d * np.log(2 * np.pi) - 0.5 * np.linalg.slogdet(H)[1]


def overlapping_2d(seed=21, n=40):
    rng = np.random.default_rng(seed)
    X = rng.normal(0, 1.5, (n, 2))
    y = (rng.random(n) < sigmoid(X @ np.array([1.4, -0.9]))).astype(float)
    return X, y


def ellipse(m, C, k=2.0, n=100):
    t = np.linspace(0, 2 * np.pi, n)
    L = np.linalg.cholesky(C)
    P = m[:, None] + k * L @ np.stack([np.cos(t), np.sin(t)])
    return P[0], P[1]


def fig_S15_1(out=None):
    """Quadratic bounds on the sigmoid: Jaakkola-Jordan (tight at +-psi) and Boehning (tight at psi)."""
    x = np.linspace(-6, 6, 400)
    psi = 2.5
    fig, axes = plt.subplots(1, 2, figsize=(8.4, 3.0), sharey=True)
    axes[0].plot(x, sigmoid(x), color=RED, label=r"$\sigma(x)$")
    axes[0].plot(x, jj_bound(x, psi), color=BLUE, ls=":", lw=2, label=r"JJ bound, $\psi=2.5$")
    axes[0].set_title("Jaakkola-Jordan", fontsize=10)
    axes[1].plot(x, sigmoid(x), color=RED, label=r"$\sigma(x)$")
    axes[1].plot(x, np.exp(-bohning_logistic_bound(-x, -psi)), color=BLUE, ls=":", lw=2, label=r"Boehning bound, $\psi=2.5$")
    axes[1].set_title("Boehning", fontsize=10)
    for i, a in enumerate(axes):
        a.legend(fontsize=8, loc="upper left"); a.set_xlabel("x"); a.set_ylim(-0.05, 1.05); panel_label(a, "ab"[i], "lower right")
        for t_ in ((psi, -psi) if i == 0 else (psi,)):
            a.axvline(t_, color=GREY, lw=0.6, ls="--")
    print("   JJ bound max violation %.2e; Boehning (as lower bound on sigma) max violation %.2e" % (np.max(jj_bound(x, psi) - sigmoid(x)), np.max(np.exp(-bohning_logistic_bound(-x, -psi)) - sigmoid(x))))
    fig.tight_layout()
    _save(fig, "figS15-1", out)


def fig_S15_2(out=None):
    """Beta-binomial regression on the Berkeley data: posterior predictive and posterior of the admission rates."""
    A, N, dept, male = berkeley()
    r = berkeley_hmc("betabin", 4000, 3)
    Q = r["samples"]
    rng = np.random.default_rng(0)
    g = 1 - male
    pi = sigmoid(Q[:, g])
    kappa = np.exp(Q[:, [2]]) + 2
    mu_draw = rng.beta(pi * kappa, (1 - pi) * kappa)
    Apred = rng.binomial(N.astype(int)[None], mu_draw)
    fig, axes = plt.subplots(1, 2, figsize=(10, 3.4))
    _berk_panel(axes[0], pi, Apred, "posterior predictive")
    axes[0].set_ylabel("admission rate")
    ax = axes[1]
    t = np.linspace(0, 1, 300)
    for col, k, lab in ((BLUE, 0, "male"), (RED, 1, "female")):
        sm = sigmoid(Q[:, k])
        ax.plot(t, stats.gaussian_kde(sm)(t), color=col, label=lab)
    ax.set_xlabel("probability of admission"); ax.set_ylabel("density"); ax.legend(fontsize=8); ax.set_title("posterior of the admission rate", fontsize=9.5)
    for i, a in enumerate(axes):
        panel_label(a, "ab"[i], "upper right")
    d = sigmoid(Q[:, 0]) - sigmoid(Q[:, 1])
    print("   beta-binomial: kappa mean %.2f, 89%% interval of the rate difference %s, P(diff>0) = %.2f (ess %.0f)" % (kappa.mean(), np.round(np.percentile(d, [5.5, 94.5]), 2), np.mean(d > 0), ess_bulk(Q[:, 0])))
    fig.tight_layout()
    _save(fig, "figS15-2", out)


def glmm_logp():
    """Varying intercept and slope model of Section S15.3 in the non-centred form (S15.87)-(S15.91).
    q = (u0, u1, log s0, log s1, z_r, z_{j,0..1} for 6 departments)."""
    import torch
    A, N, dept, male = berkeley()
    At, Nt, dp, ml = torch.tensor(A), torch.tensor(N), torch.tensor(dept), torch.tensor(male, dtype=torch.float64)

    def logp(q):
        u0, u1 = q[0], q[1]
        s = torch.exp(q[2:4])
        r = torch.tanh(q[4])
        Z = q[5:].reshape(6, 2)
        L = torch.stack([torch.stack([torch.ones(()), torch.zeros(())]), torch.stack([r, torch.sqrt(1 - r ** 2)])])
        V = (Z @ L.T) * s                        # rows: sigma L z_j
        eta = u0 + V[dp, 0] + (u1 + V[dp, 1]) * ml
        ll = torch.sum(At * eta - Nt * torch.nn.functional.softplus(eta))
        pri = -0.5 * (u0 / 4) ** 2 - 0.5 * (u1 / 1) ** 2 - 0.5 * torch.sum(s ** 2) + torch.sum(q[2:4])
        lkj = 2 * torch.log(1 - r ** 2)          # LKJ(2) in 2-d: (1 - r^2) times the Jacobian (1 - r^2)
        return ll + pri + lkj - 0.5 * torch.sum(Z ** 2)
    return logp


def fig_S15_3(out=None):
    """Varying intercept and slope GLMM for the Berkeley data: posterior predictive check."""
    A, N, dept, male = berkeley()
    rng = np.random.default_rng(0)
    q0 = np.zeros(17)
    r = hmc(glmm_logp(), q0, n_samples=2000, n_warm=600, seed=4, delta=0.9, traj=1.5)
    Q = r["samples"]
    S = len(Q)
    s = np.exp(Q[:, 2:4]); rho = np.tanh(Q[:, 4]); Z = Q[:, 5:].reshape(S, 6, 2)
    v0 = s[:, [0]] * Z[:, :, 0]
    v1 = s[:, [1]] * (rho[:, None] * Z[:, :, 0] + np.sqrt(1 - rho ** 2)[:, None] * Z[:, :, 1])
    eta = Q[:, [0]] + v0[:, dept] + (Q[:, [1]] + v1[:, dept]) * male[None]
    mu = sigmoid(eta)
    Apred = rng.binomial(N.astype(int)[None], mu)
    fig, ax = plt.subplots(figsize=(5.4, 3.4))
    _berk_panel(ax, mu, Apred, "varying intercept and slope")
    ax.set_ylabel("admission rate")
    print("   GLMM: posterior mean u = %s, sigma = %s, rho = %.2f; divergences %d, eps %.3f, ESS(u1) %.0f" % (np.round(Q[:, :2].mean(0), 2), np.round(s.mean(0), 2), rho.mean(), r["divergences"], r["eps"], ess_bulk(Q[:, 1])))
    fig.tight_layout()
    _save(fig, "figS15-3", out)


# ============================================================================== sparse priors (15.2.5 - 15.2.8)
def ard_fit(X, y, method="mackay", n_iter=500, alpha0=1.0, prune=1e10):
    """Automatic relevancy determination: type-II maximum likelihood for the weight precisions alpha_j and the
    noise precision beta. method 'mackay' (fixed-point, MacKay 1992) or 'em' (EM). Returns dict with the posterior
    mean and covariance, alpha, beta and the history of log10 alpha and of the log marginal likelihood."""
    N, D = X.shape
    alpha = np.full(D, alpha0)
    beta = 1.0 / (0.1 * np.var(y))
    ha, hl = [], []
    for _ in range(n_iter):
        A = np.diag(alpha)
        Sig = np.linalg.inv(beta * X.T @ X + A)
        mu = beta * Sig @ X.T @ y
        if method == "mackay":
            gam = 1 - alpha * np.diag(Sig)
            alpha = np.minimum(gam / np.maximum(mu ** 2, 1e-300), prune)
            beta = (N - gam.sum()) / np.sum((y - X @ mu) ** 2)
        else:
            alpha = np.minimum(1.0 / (mu ** 2 + np.diag(Sig)), prune)
            beta = N / (np.sum((y - X @ mu) ** 2) + np.trace(X @ Sig @ X.T))
        ha.append(np.log10(alpha)); hl.append(ard_log_evidence(X, y, alpha, beta))
    A = np.diag(alpha)
    Sig = np.linalg.inv(beta * X.T @ X + A)
    mu = beta * Sig @ X.T @ y
    return dict(mu=mu, Sigma=Sig, alpha=alpha, beta=beta, hist_alpha=np.array(ha), hist_ll=np.array(hl))


def ard_log_evidence(X, y, alpha, beta):
    """log N(y | 0, beta^{-1} I + X diag(1/alpha) X^T), (15.99)."""
    C = np.eye(len(y)) / beta + (X / alpha) @ X.T
    sign, ld = np.linalg.slogdet(C)
    return -0.5 * (len(y) * np.log(2 * np.pi) + ld + y @ np.linalg.solve(C, y))


def blasso_gibbs(X, y, gamma, n=2000, seed=0):
    """Bayesian lasso Gibbs sampler (Park and Casella 2008) for (15.90)-(15.92), p(sigma^2) proportional to 1/sigma^2."""
    rng = np.random.default_rng(seed)
    N, D = X.shape
    XtX, Xty = X.T @ X, X.T @ y
    w = np.zeros(D); s2 = np.var(y); itau = np.ones(D)
    out = np.empty((n, D))
    for t in range(n):
        A = XtX + np.diag(itau)
        Ai = np.linalg.inv(A)
        w = rng.multivariate_normal(Ai @ Xty, s2 * Ai)
        rss = np.sum((y - X @ w) ** 2)
        s2 = 1.0 / rng.gamma((N + D) / 2, 1.0 / (0.5 * (rss + np.sum(itau * w ** 2))))
        mu_ig = np.sqrt(gamma ** 2 * s2 / np.maximum(w ** 2, 1e-300))
        itau = rng.wald(mu_ig, gamma ** 2)
        out[t] = w
    return out


def horseshoe_gibbs(X, y, n=2000, seed=0):
    """Horseshoe Gibbs sampler with inverse-gamma auxiliary variables (Makalic and Schmidt 2016) for (15.95)-(15.97)."""
    rng = np.random.default_rng(seed)
    N, D = X.shape
    XtX, Xty = X.T @ X, X.T @ y
    lam2 = np.ones(D); nu = np.ones(D); tau2 = 1.0; xi = 1.0; s2 = np.var(y)
    out = np.empty((n, D))
    ig = lambda a, b: 1.0 / rng.gamma(a, 1.0 / b)
    for t in range(n):
        A = XtX + np.diag(1.0 / (tau2 * lam2))
        Ai = np.linalg.inv(A)
        w = rng.multivariate_normal(Ai @ Xty, s2 * Ai)
        rss = np.sum((y - X @ w) ** 2)
        s2 = ig((N + D) / 2, 0.5 * rss + 0.5 * np.sum(w ** 2 / (tau2 * lam2)))
        lam2 = np.array([ig(1.0, 1.0 / nu[j] + w[j] ** 2 / (2 * tau2 * s2)) for j in range(D)])
        tau2 = ig((D + 1) / 2, 1.0 / xi + np.sum(w ** 2 / lam2) / (2 * s2))
        nu = np.array([ig(1.0, 1.0 + 1.0 / lam2[j]) for j in range(D)])
        xi = ig(1.0, 1.0 + 1.0 / tau2)
        out[t] = w
    return out


def mniw_posterior(X, Y, M0, V0, nu0, Psi0):
    """Matrix-normal inverse-Wishart update (15.118)-(15.125): Y is Ny x N, X is Nx x N, W is Ny x Nx,
    W | Sigma ~ MN(M0, Sigma, V0^{-1}) (V0 acts as a precision), Sigma ~ IW(nu0, Psi0)."""
    Sxx = X @ X.T + V0
    Syx = Y @ X.T + M0 @ V0
    Syy = Y @ Y.T + M0 @ V0 @ M0.T
    Syx_ = Syx @ np.linalg.solve(Sxx, Syx.T)
    M1 = Syx @ np.linalg.inv(Sxx)
    return M1, Sxx, nu0 + X.shape[1], Psi0 + Syy - Syx_


def ard_data(seed=8, N=30, D=10):
    rng = np.random.default_rng(seed)
    X = rng.standard_normal((N, D))
    w = np.zeros(D); w[[0, 3, 6]] = [2.5, -1.8, 1.2]
    y = X @ w + rng.normal(0, 1.0, N)
    return X, y, w


# -- shrinkage profiles of one coefficient (Figure 15.X1)
def horseshoe_marginal(w, tau, ng=4001):
    """Marginal prior density of w when w | gamma ~ N(0, gamma^2 tau^2), gamma ~ C+(0,1), by quadrature in log gamma."""
    lg = np.linspace(-14, 14, ng)
    g = np.exp(lg)
    wg = (2 / np.pi) / (1 + g ** 2) * g * (lg[1] - lg[0])
    return (stats.norm.pdf(np.asarray(w)[:, None], 0, g[None] * tau) * wg[None]).sum(1)


def shrink_profiles(y_grid, tau_g=1.5, b_lap=1.0, tau_h=0.5, pi=0.2, s_slab=3.0):
    """Posterior mean of w for y | w ~ N(w, 1) under four priors, by quadrature (spike and slab in closed form)."""
    wg = np.linspace(-40, 40, 16001)
    dw = wg[1] - wg[0]
    priors = {"gauss": stats.norm.pdf(wg, 0, tau_g), "laplace": np.exp(-np.abs(wg) / b_lap) / (2 * b_lap), "horseshoe": horseshoe_marginal(wg, tau_h)}
    out = {}
    for k, pw in priors.items():
        m = np.empty(len(y_grid))
        for i, yv in enumerate(y_grid):
            lik = stats.norm.pdf(yv, wg, 1.0) * pw
            m[i] = np.sum(wg * lik) / np.sum(lik)
        out[k] = m
    pm = pi * stats.norm.pdf(y_grid, 0, np.sqrt(1 + s_slab ** 2))
    p0 = (1 - pi) * stats.norm.pdf(y_grid, 0, 1.0)
    out["spike"] = pm / (pm + p0) * s_slab ** 2 / (1 + s_slab ** 2) * y_grid
    return out, priors, wg


def fig_15_X1(out=None):
    """Shrinkage by the Gaussian, Laplace, horseshoe and spike-and-slab priors; the horseshoe shrinkage weight."""
    yg = np.linspace(-9, 9, 181)
    prof, priors, wg = shrink_profiles(yg)
    fig, axes = plt.subplots(1, 3, figsize=(12, 3.3))
    ax = axes[0]
    cols = dict(gauss=GREY, laplace=BLUE, horseshoe=RED)
    nm = dict(gauss="Gaussian", laplace="Laplace", horseshoe="horseshoe")
    for k in ("gauss", "laplace", "horseshoe"):
        ax.semilogy(wg, priors[k], color=cols[k], label=nm[k])
    ax.set_xlim(-6, 6); ax.set_ylim(1e-4, 5); ax.set_xlabel("$w$"); ax.set_ylabel("prior density"); ax.legend(fontsize=8)
    ax = axes[1]
    ax.plot(yg, yg, color="k", lw=0.8, ls="--", label="no shrinkage")
    for k, c in (("gauss", GREY), ("laplace", BLUE), ("horseshoe", RED), ("spike", GREEN)):
        ax.plot(yg, prof[k], color=c, label={"gauss": "Gaussian", "laplace": "Laplace", "horseshoe": "horseshoe", "spike": "spike and slab"}[k])
    ax.set_xlabel("observation $y$"); ax.set_ylabel("$\\mathbb{E}[w\\mid y]$"); ax.legend(fontsize=8)
    ax = axes[2]
    rng = np.random.default_rng(0)
    gam = np.abs(rng.standard_cauchy(200000))
    kap = 1 / (1 + gam ** 2)
    ax.hist(kap, bins=np.linspace(0, 1, 41), density=True, color=BLUE, alpha=0.5, label="simulated")
    t = np.linspace(0.005, 0.995, 300)
    ax.plot(t, stats.beta.pdf(t, 0.5, 0.5), color=RED, label="Beta(1/2, 1/2)")
    ax.set_ylim(0, 5); ax.set_xlabel(r"shrinkage weight $\kappa=1/(1+\gamma^2)$"); ax.legend(fontsize=8)
    for i, a in enumerate(axes):
        panel_label(a, "abc"[i], "upper center" if i == 2 else "lower right")
    print("   E[w|y=6]: Gauss %.2f, Laplace %.2f, horseshoe %.2f, spike %.2f" % tuple(prof[k][np.argmin(np.abs(yg - 6))] for k in ("gauss", "laplace", "horseshoe", "spike")))
    fig.tight_layout()
    _save(fig, "fig15-X1", out)


def fig_15_X2(out=None):
    """ARD: marginal-likelihood geometry (cf. Figure 15.4), and ARD against ridge and lasso on a sparse regression."""
    from sklearn.linear_model import RidgeCV, LassoCV
    fig, axes = plt.subplots(2, 3, figsize=(12, 6.2))
    y2 = np.array([1.0, 0.25]); beta = 4.0
    xo = np.array([0.25, -1.0]); xa = np.array([1.0, 0.25])
    g = np.linspace(-2.2, 2.2, 200); G1, G2 = np.meshgrid(g, g)
    P = np.stack([G1.ravel(), G2.ravel()], 1)
    for ax, alpha, ttl in ((axes[0, 0], 0.3, r"finite $\alpha$ (feature $\perp y$)"), (axes[0, 1], np.inf, r"$\alpha=\infty$")):
        C = np.eye(2) / beta + (0 if np.isinf(alpha) else np.outer(xo, xo) / alpha)
        Z = np.exp(common.gauss_logpdf(P, np.zeros(2), C)).reshape(G1.shape)
        ax.contour(G1, G2, Z, levels=8, colors=BLUE, linewidths=0.9)
        ax.plot(*y2, "o", color=RED); ax.annotate("$y$", y2, xytext=(5, 5), textcoords="offset points", color=RED)
        ax.arrow(0, 0, *xo, color="k", head_width=0.06, length_includes_head=True); ax.text(xo[0] + 0.05, xo[1], "$x$", fontsize=9)
        ax.set_aspect("equal"); ax.set_title(ttl, fontsize=9.5); ax.set_xlim(-2.2, 2.2); ax.set_ylim(-2.2, 2.2)
    ax = axes[0, 2]
    la = np.linspace(-6, 6, 300)
    for xv, c, lab in ((xo, RED, "$x \\perp y$ (irrelevant)"), (xa, BLUE, "$x \\parallel y$ (relevant)")):
        ll = [common.gauss_logpdf(y2[None], np.zeros(2), np.eye(2) / beta + np.outer(xv, xv) * 10 ** (-l))[0] for l in la]
        ax.plot(la, ll, color=c, label=lab)
    ax.set_xlabel(r"$\log_{10}\alpha$"); ax.set_ylabel(r"$\log p(y\mid\alpha,\beta)$"); ax.legend(fontsize=8)
    X, y, wtrue = ard_data()
    ard = ard_fit(X, y, "mackay", 1000)
    ridge = RidgeCV(alphas=np.logspace(-2, 3, 60), fit_intercept=False).fit(X, y)
    lasso = LassoCV(cv=5, fit_intercept=False, random_state=0).fit(X, y)
    ols = np.linalg.lstsq(X, y, rcond=None)[0]
    ax = axes[1, 0]
    D = X.shape[1]; w_ = 0.2
    for k, (c, v, lab) in enumerate(((GREY, ols, "least squares"), (BLUE, ridge.coef_, "ridge"), (GREEN, lasso.coef_, "lasso"), (RED, ard["mu"], "ARD"))):
        ax.bar(np.arange(D) + (k - 1.5) * w_, v, width=w_, color=c, label=lab)
    ax.plot(np.arange(D), wtrue, "k_", ms=14, mew=1.5, label="truth")
    ax.set_xlabel("coefficient index"); ax.legend(fontsize=7, ncol=2)
    ax = axes[1, 1]
    ax.plot(ard["hist_alpha"][:200], lw=1)
    ax.set_xlabel("iteration"); ax.set_ylabel(r"$\log_{10}\alpha_j$"); ax.set_title("precisions of the 10 features", fontsize=9.5)
    ax = axes[1, 2]
    ax.plot(ard["hist_ll"], color=RED, label="MacKay")
    em = ard_fit(X, y, "em", 1000)
    ax.plot(em["hist_ll"], color=BLUE, label="EM")
    ax.set_xscale("log"); ax.set_xlabel("iteration"); ax.set_ylabel("log marginal likelihood"); ax.legend(fontsize=8)
    for i, a in enumerate(axes.ravel()):
        panel_label(a, "abcdef"[i], "lower right")
    nz = np.sum(np.abs(ard["mu"]) > 1e-3)
    print("   ARD non-negligible coefficients: %d (truth 3), lasso %d, ridge %d; ARD error %.3f, ridge %.3f, lasso %.3f, ols %.3f" % (nz, np.sum(lasso.coef_ != 0), np.sum(np.abs(ridge.coef_) > 1e-3), np.linalg.norm(ard["mu"] - wtrue), np.linalg.norm(ridge.coef_ - wtrue), np.linalg.norm(lasso.coef_ - wtrue), np.linalg.norm(ols - wtrue)))
    fig.tight_layout()
    _save(fig, "fig15-X2", out)
    _dump("ard", dict(note="Simulated sparse regression for the ARD demo: N=30, D=10, three non-zero coefficients, noise sd 1.",
                      X=np.round(X, 5).tolist(), y=np.round(y, 5).tolist(), w=wtrue.tolist(),
                      ref_mackay=dict(alpha_log10=np.round(np.log10(ard["alpha"]), 4).tolist(), mu=np.round(ard["mu"], 5).tolist(), beta=float(ard["beta"]))))


def fig_15_X3(out=None):
    """Variational (JJ, Boehning), Laplace and exact posteriors of a two-parameter logistic model."""
    X, y = overlapping_2d()
    sd = 5.0
    ev, m_ex, C_ex, (W1, W2, Pg) = exact_logistic_posterior_2d(X, y, sd)
    wl, H = logreg_map(X, y, sd ** 2)
    mj, Vj, _, hj = jj_vi(X, y, np.zeros(2), sd ** 2 * np.eye(2), 60)
    mb, Vb, _, hb = bohning_vi(X, y, np.zeros(2), sd ** 2 * np.eye(2), 200)
    ev_l = laplace_log_evidence(X, y, sd ** 2)
    fig, axes = plt.subplots(1, 2, figsize=(9.6, 3.7))
    ax = axes[0]
    ax.contour(W1, W2, Pg, levels=6, colors=GREY, linewidths=0.8)
    for (m, C, c, lab) in ((m_ex, C_ex, "k", "exact (2 sd)"), (wl, np.linalg.inv(H), GREEN, "Laplace"), (mj, Vj, RED, "JJ"), (mb, Vb, BLUE, "Boehning")):
        ax.plot(*ellipse(m, C), color=c, lw=1.6, label=lab)
    ax.set_xlim(-0.5, 4.5); ax.set_ylim(-4, 1); ax.set_xlabel("$w_1$"); ax.set_ylabel("$w_2$"); ax.legend(fontsize=8)
    ax = axes[1]
    ax.plot(np.arange(1, len(hj) + 1), hj, color=RED, label="JJ bound")
    ax.plot(np.arange(1, len(hb) + 1), hb, color=BLUE, label="Boehning bound")
    ax.axhline(ev, color="k", lw=1, label="exact log evidence")
    ax.axhline(ev_l, color=GREEN, lw=1, ls="--", label="Laplace estimate")
    ax.set_xscale("log"); ax.set_xlabel("iteration"); ax.set_ylabel("lower bound on $\\log p(\\mathcal{D})$"); ax.legend(fontsize=8)
    for i, a in enumerate(axes):
        panel_label(a, "ab"[i], "lower right")
    print("   log evidence: exact %.3f, Laplace %.3f, JJ %.3f, Boehning %.3f; means exact %s Laplace %s JJ %s B %s" % (ev, ev_l, hj[-1], hb[-1], np.round(m_ex, 3), np.round(wl, 3), np.round(mj, 3), np.round(mb, 3)))
    fig.tight_layout()
    _save(fig, "fig15-X3", out)


FIGURES = {"15.1": fig_15_1, "15.2": fig_15_2, "15.5": fig_15_5, "15.6": fig_15_6, "15.7": fig_15_7, "15.8": fig_15_8,
           "15.9": fig_15_9, "15.11": fig_15_11, "15.12": fig_15_12, "15.15": fig_15_15, "15.16": fig_15_16, "15.17": fig_15_17,
           "S15.1": fig_S15_1, "S15.2": fig_S15_2, "S15.3": fig_S15_3, "15.X1": fig_15_X1, "15.X2": fig_15_X2, "15.X3": fig_15_X3}
if __name__ == "__main__":
    common.run(FIGURES, CH)
