"""Murphy (2023), Chapter 24: Energy-based models (and Supplement Chapter 24).

Companion code for Chapter 24 of K. P. Murphy, Probabilistic Machine Learning: Advanced Topics
(MIT Press, 2023; December 2025 draft), pp. 849-865, to be read next to the lecture notes
(../ch24-energy-based-models.html). The supplement chapter 24 has no sections in this draft. Written
from scratch with numpy, matplotlib and PyTorch (CPU only); every run is seeded and takes seconds.

Methods
* Section 24.1.1: a product of two experts as a sum of energies, sampled with Langevin dynamics
  (expert_A, expert_B, poe_energy, ula, mala) -> Figure 24.1 (own energies in the same spirit).
* Section 24.2: the gradient of the log partition function as a model expectation, checked on a
  one-dimensional polynomial energy (fit_mle_poly) and on a restricted Boltzmann machine small
  enough to enumerate (class RBMEnum): exact maximum likelihood, contrastive divergence with
  k Gibbs steps (expected update, no sampling noise) and persistent contrastive divergence.
  Section 24.2.1: unadjusted and Metropolis-adjusted Langevin sampling and the exact stationary
  variance of the discretised chain on a Gaussian. Section 24.2.2.1: Algorithm 24.1 on the 8 x 8
  digits of scikit-learn (train_rbm_digits), with CD-1 and persistent chains.
* Section 24.3: score matching (closed form for an energy that is linear in its parameters),
  denoising score matching, sliced score matching and their variances (fit_sm_poly, dsm_*), and a
  small neural energy trained by denoising and by sliced score matching on a 2-D swiss roll
  (the example the text points to in its Figure 25.5; the figure here is my own).
* Section 24.4: noise contrastive estimation with a learned normalising constant (fit_nce_poly,
  nce_logz_rmse).

Skipped figures, with reasons: Figure 24.2 (schematic of contrastive divergence sampling in an RBM).

Own figures (not in the book): x24.a to x24.g here are Figures 24.x1, 24.x4... in the notes in this order:
x24.a = 24.x1 (Langevin bias), x24.c = 24.x2 (enumerable RBM), x24.d = 24.x3 (RBM on digits), x24.b = 24.x4 (1-D estimators),
x24.e = 24.x5 (DSM bias and variance), x24.g = 24.x6 (swiss roll), x24.f = 24.x7 (NCE noise).

Data: simulation, and the 8 x 8 digits bundled with scikit-learn (stand-in for the book's own
example data; the book's Figure 24.1 is a schematic of two energies and uses no data set).
"""
import itertools
import os
import sys

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import common  # noqa: E402
from common import BLUE, CYAN, GREEN, GREY, ORANGE, PURPLE, RED, MAGENTA, plt, savefig, panel_grid, panel_label  # noqa: E402

CH = 24
CACHE_DIR = os.path.join(HERE, "cache", "ch24")
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
os.makedirs(CACHE_DIR, exist_ok=True)


def sigm(a):
    return common.sigmoid(a)


def softplus(a):
    return np.logaddexp(0.0, a)


# =========================================================================== Section 24.1.1: product of experts
POE_CENTRES = np.array([[1.0, 1.0], [-1.0, 1.0], [-1.0, -1.0], [1.0, -1.0]])
POE_S = 0.45        # width of each well of expert A
POE_KAPPA = 5.0     # sharpness of the half-plane expert B


def expert_A(x):
    """Energy of expert A: minus the log of an equal mixture of four Gaussian wells (SD 0.45)."""
    x = np.atleast_2d(x)
    d2 = ((x[:, None, :] - POE_CENTRES[None]) ** 2).sum(-1)
    return -common.logsumexp(-d2 / (2 * POE_S ** 2), axis=1) + np.log(4.0)


def grad_A(x):
    x = np.atleast_2d(x)
    d2 = ((x[:, None, :] - POE_CENTRES[None]) ** 2).sum(-1)
    lw = -d2 / (2 * POE_S ** 2)
    r = np.exp(lw - common.logsumexp(lw, axis=1, keepdims=True))
    return (r[:, :, None] * (x[:, None, :] - POE_CENTRES[None])).sum(1) / POE_S ** 2


def expert_B(x):
    """Energy of expert B: softplus(-kappa x_1), the soft constraint 'the first coordinate is positive'."""
    x = np.atleast_2d(x)
    return softplus(-POE_KAPPA * x[:, 0])


def grad_B(x):
    x = np.atleast_2d(x)
    g = np.zeros_like(x)
    g[:, 0] = -POE_KAPPA * sigm(-POE_KAPPA * x[:, 0])
    return g


def poe_energy(x, use=("A", "B")):
    """Energy of the product of experts (24.4): the sum of the energies of the experts used."""
    e = 0.0
    if "A" in use:
        e = e + expert_A(x)
    if "B" in use:
        e = e + expert_B(x)
    return e


def poe_grad(x, use=("A", "B")):
    g = 0.0
    if "A" in use:
        g = g + grad_A(x)
    if "B" in use:
        g = g + grad_B(x)
    return g


def ula(grad_E, x0, eps, K, rng, keep=False):
    """Unadjusted Langevin (24.18): x <- x - (eps^2 / 2) grad E(x) + eps z. x0 has one chain per row."""
    x = np.array(x0, float)
    path = [x.copy()] if keep else None
    for _ in range(K):
        x = x - 0.5 * eps ** 2 * grad_E(x) + eps * rng.standard_normal(x.shape)
        if keep:
            path.append(x.copy())
    return (x, np.array(path)) if keep else x


def mala(E, grad_E, x0, eps, K, rng):
    """Metropolis-adjusted Langevin: the proposal of ula, accepted with the Metropolis-Hastings ratio.
    Returns the final states and the mean acceptance rate."""
    x = np.array(x0, float)
    acc = 0.0
    for _ in range(K):
        gx = grad_E(x)
        y = x - 0.5 * eps ** 2 * gx + eps * rng.standard_normal(x.shape)
        gy = grad_E(y)
        lq_fwd = -((y - x + 0.5 * eps ** 2 * gx) ** 2).sum(1) / (2 * eps ** 2)
        lq_bwd = -((x - y + 0.5 * eps ** 2 * gy) ** 2).sum(1) / (2 * eps ** 2)
        la = -E(y) + E(x) + lq_bwd - lq_fwd
        ok = np.log(rng.random(len(x))) < la
        x = np.where(ok[:, None], y, x)
        acc += ok.mean()
    return x, acc / K


def ula_gaussian_variance(a, eps):
    """Stationary variance of x <- (1 - eps^2 a / 2) x + eps z (energy a x^2 / 2): 1 / (a (1 - eps^2 a / 4)),
    finite for eps^2 a < 4 (gap 24.3)."""
    return 1.0 / (a * (1.0 - eps ** 2 * a / 4.0))


def poe_grid_density(use, lim=2.6, n=260):
    g = np.linspace(-lim, lim, n)
    X, Y = np.meshgrid(g, g)
    P = np.stack([X.ravel(), Y.ravel()], 1)
    E = poe_energy(P, use)
    d = np.exp(-(E - E.min())).reshape(n, n)
    return g, d / (d.sum() * (g[1] - g[0]) ** 2)


def poe_right_mass(use=("A", "B")):
    """Probability that x_1 > 0 under the product of experts (by quadrature)."""
    g, d = poe_grid_density(use, lim=4.0, n=641)
    h = g[1] - g[0]
    X = np.meshgrid(g, g)[0]
    return float((d * ((X > 0) + 0.5 * (X == 0))).sum() * h * h)


def fig_24_1(out=None):
    """Figure 24.1: two energies in 2-D and their sum, with Langevin trajectories."""
    rng = np.random.default_rng(241)
    fig, axes = panel_grid(1, 3, size=(2.7, 2.7))
    starts = rng.uniform(-2.2, 2.2, size=(7, 2))
    titles = {("A",): "energy A", ("B",): "energy B", ("A", "B"): "energy A + B"}
    for ax, use in zip(axes[0], titles):
        g, d = poe_grid_density(use)
        ax.imshow(-np.log(d + 1e-12), origin="lower", extent=[g[0], g[-1], g[0], g[-1]], cmap="viridis_r", vmin=-1, vmax=9)
        ax.contour(g, g, -np.log(d + 1e-12), levels=[1.0, 3.0], colors="white", linewidths=0.4, alpha=0.6)
        r2 = np.random.default_rng(7)
        _, path = ula(lambda z, u=use: poe_grad(z, u), starts, 0.3, 70, r2, keep=True)
        for c in range(path.shape[1]):
            ax.plot(path[:, c, 0], path[:, c, 1], color="k", lw=0.7, alpha=0.85)
            ax.plot(path[-1, c, 0], path[-1, c, 1], "o", color="white", mec="k", ms=3.3, mew=0.8)
            ax.plot(path[0, c, 0], path[0, c, 1], "x", color="k", ms=3.5, mew=0.8)
        ax.set_title(titles[use])
        ax.set_xticks([-2, 0, 2])
        ax.set_yticks([-2, 0, 2])
        ax.set_xlim(-2.6, 2.6)
        ax.set_ylim(-2.6, 2.6)
    savefig(fig, CH, "fig24-1", out)


# =========================================================================== Section 24.2.1: Langevin and MALA
def x24a_data(seed=11):
    """Variance of ULA and MALA on a standard Gaussian target against the step size eps, and the MALA acceptance rate."""
    rng = np.random.default_rng(seed)
    eps_list = np.array([0.1, 0.3, 0.6, 0.9, 1.2, 1.5, 1.8])
    rows = []
    for eps in eps_list:
        x0 = rng.standard_normal((4000, 1))
        K = 300
        xu = ula(lambda z: z, x0, eps, K, rng)
        xm, acc = mala(lambda z: 0.5 * (z ** 2).sum(1), lambda z: z, x0, eps, K, rng)
        rows.append((eps, xu.var(), ula_gaussian_variance(1.0, eps), xm.var(), acc))
    return np.array(rows)


def fig_x24_a(out=None):
    """Own figure: discretisation bias of unadjusted Langevin sampling and its MALA correction."""
    D = x24a_data()
    fig, axes = panel_grid(1, 2, size=(3.3, 2.6))
    ax = axes[0, 0]
    e = np.linspace(0.05, 1.95, 200)
    ax.plot(e, ula_gaussian_variance(1.0, e), color=RED, lw=1.4, label=r"theory $1/(1-\epsilon^2/4)$")
    ax.plot(D[:, 0], D[:, 1], "o", color=RED, ms=4, label="ULA, simulated")
    ax.plot(D[:, 0], D[:, 3], "s", color=BLUE, ms=4, label="MALA, simulated")
    ax.axhline(1.0, color=GREY, lw=0.8, ls="--")
    ax.set_ylim(0.8, 4.5)
    ax.set_xlabel(r"step size $\epsilon$")
    ax.set_ylabel("variance of the sample")
    ax.legend(loc="upper left", frameon=False)
    panel_label(ax, "a", "upper right")
    ax = axes[0, 1]
    ax.plot(D[:, 0], D[:, 4], "s-", color=BLUE, ms=4)
    ax.set_xlabel(r"step size $\epsilon$")
    ax.set_ylabel("MALA acceptance rate")
    ax.set_ylim(0, 1.02)
    panel_label(ax, "b", "lower left")
    savefig(fig, CH, "figx24-a", out)


# =========================================================================== 1-D polynomial energy: estimators
GRID = np.linspace(-6.0, 6.0, 1201)
DATASETS = {
    "bimodal": dict(w=[0.5, 0.5], m=[-1.5, 1.5], s=[0.6, 0.6], n=400, seed=24),
    "skewed": dict(w=[0.7, 0.3], m=[-0.8, 1.4], s=[0.7, 0.5], n=400, seed=25),
}


def mix_sample(name):
    d = DATASETS[name]
    rng = np.random.default_rng(d["seed"])
    k = rng.choice(len(d["w"]), size=d["n"], p=d["w"])
    return rng.normal(np.array(d["m"])[k], np.array(d["s"])[k])


def mix_pdf(x, name):
    d = DATASETS[name]
    x = np.asarray(x, float)
    return sum(w * np.exp(-0.5 * ((x - m) / s) ** 2) / (s * np.sqrt(2 * np.pi)) for w, m, s in zip(d["w"], d["m"], d["s"]))


def phi(x):
    x = np.asarray(x, float)
    return np.stack([x, x ** 2, x ** 3, x ** 4], -1)


def dphi(x):
    x = np.asarray(x, float)
    return np.stack([np.ones_like(x), 2 * x, 3 * x ** 2, 4 * x ** 3], -1)


def ddphi(x):
    x = np.asarray(x, float)
    return np.stack([np.zeros_like(x), 2 * np.ones_like(x), 6 * x, 12 * x ** 2], -1)


def _gauss_weights(lo, hi, n):
    z = np.linspace(lo, hi, n)
    w = np.exp(-0.5 * z ** 2)
    return z, w / w.sum()


def poly_density(theta):
    """Density exp(-theta . phi(x)) / Z on the window [-6, 6] (grid GRID) and log Z."""
    lp = -(phi(GRID) @ np.asarray(theta, float))
    m = lp.max()
    lz = np.log(np.trapz(np.exp(lp - m), GRID)) + m
    return np.exp(lp - lz), lz


def fit_sm_poly(x, sigma=0.0):
    """Score matching (sigma = 0) and denoising score matching (sigma > 0) for E = theta . phi(x).

    The score is -theta . phi'(x), so the implicit objective (24.30) is quadratic in theta:
    (1/2) theta' A theta - b' theta with A = E[phi' phi'^T] and b = E[phi'']; the minimiser is
    A^{-1} b. With noise the expectation is over the noisy points x + sigma z (Vincent's identity
    makes DSM score matching on the smoothed data); z is integrated on a grid with Gaussian weights.
    """
    x = np.asarray(x, float)
    if sigma > 0:
        zz, ww = _gauss_weights(-6.0, 6.0, 161)
        X = (x[:, None] + sigma * zz[None, :]).ravel()
        W = (np.tile(ww, len(x)) / len(x))
    else:
        X, W = x, np.full(len(x), 1.0 / len(x))
    D = dphi(X)
    A = (D * W[:, None]).T @ D
    b = (ddphi(X) * W[:, None]).sum(0)
    return np.linalg.solve(A, b)


def fit_mle_poly(x, iters=100):
    """Maximum likelihood for E = theta . phi(x) on the window [-6, 6]: Newton with backtracking.
    The negative log-likelihood is convex (exponential family): gradient E_data[phi] - E_model[phi],
    Hessian Cov_model(phi) (gap 24.2)."""
    Pm = phi(x).mean(0)
    F = phi(GRID)
    th = np.array([0.0, 0.5, 0.0, 0.01])

    def nll(t):
        return Pm @ t + poly_density(t)[1]

    for _ in range(iters):
        p, _ = poly_density(th)
        m = np.trapz(F * p[:, None], GRID, axis=0)
        C = np.trapz(F[:, :, None] * F[:, None, :] * p[:, None, None], GRID, axis=0) - np.outer(m, m)
        step = np.linalg.solve(C, Pm - m)
        f0, a = nll(th), 1.0
        while nll(th - a * step) > f0 and a > 1e-10:
            a /= 2
        th = th - a * step
        if np.abs(a * step).max() < 1e-12:
            break
    return th


def fit_nce_poly(x, m, s, nu=1.0, iters=200):
    """Noise contrastive estimation (24.46)-(24.50) for E = theta . phi(x) with a learned constant c = -log Z.
    Noise N(m, s^2), nu = p(y=1)/p(y=0). The classifier logit is log nu + log p_theta(x) - log p_n(x),
    linear in (theta, c): a logistic regression, solved by damped Newton. The noise expectation is
    taken on a fine grid with Gaussian weights. Returns (theta (4,), c)."""
    zz, ww = _gauss_weights(-8.0, 8.0, 1001)
    zn = m + s * zz

    def lpn(t):
        return -0.5 * ((t - m) / s) ** 2 - np.log(s * np.sqrt(2 * np.pi))

    x = np.asarray(x, float)
    Fd = np.hstack([-phi(x), np.ones((len(x), 1))])
    od = np.log(nu) - lpn(x)
    Fn = np.hstack([-phi(zn), np.ones((len(zn), 1))])
    on = np.log(nu) - lpn(zn)
    q1, q0 = nu / (1 + nu), 1 / (1 + nu)
    th = np.zeros(5)
    th[1] = -0.5

    def obj(t):
        return q1 * np.mean(softplus(-(Fd @ t + od))) + q0 * np.sum(ww * softplus(Fn @ t + on))

    for _ in range(iters):
        sd, sn = sigm(Fd @ th + od), sigm(Fn @ th + on)
        g = q1 * (-(1 - sd)[:, None] * Fd).mean(0) + q0 * (ww * sn) @ Fn
        H = q1 * ((sd * (1 - sd))[:, None] * Fd).T @ Fd / len(x) + q0 * (Fn * (ww * sn * (1 - sn))[:, None]).T @ Fn
        st = np.linalg.solve(H + 1e-10 * np.eye(5), g)
        f0, a = obj(th), 1.0
        while obj(th - a * st) > f0 + 1e-15 and a > 1e-10:
            a /= 2
        th = th - a * st
        if np.abs(a * st).max() < 1e-12:
            break
    return th[:4], th[4]


def kl_to_truth(name, theta):
    p = mix_pdf(GRID, name)
    q = poly_density(theta)[0]
    return float(np.trapz(p * (np.log(np.maximum(p, 1e-300)) - np.log(np.maximum(q, 1e-300))), GRID))


def estimator_table(name):
    """Fit all estimators of Section 24.2-24.4 to a data set; returns a dict name -> theta."""
    x = mix_sample(name)
    res = {"mle": fit_mle_poly(x), "sm": fit_sm_poly(x)}
    for sg in (0.2, 0.5, 1.0):
        res["dsm%.1f" % sg] = fit_sm_poly(x, sg)
    for s in (0.3, 1.0, 3.0, 10.0):
        th, c = fit_nce_poly(x, 0.0, s)
        res["nce%g" % s] = th
        res["nce%g_c" % s] = c
    return res


def fig_x24_b(out=None):
    """Own figure: the 1-D polynomial energy fitted by maximum likelihood, score matching, denoising score matching and NCE."""
    fig, axes = panel_grid(1, 2, size=(3.6, 2.7))
    for ax, name, lab in zip(axes[0], ("bimodal", "skewed"), "ab"):
        x = mix_sample(name)
        res = estimator_table(name)
        ax.hist(x, bins=36, range=(-4, 4), density=True, color="#dddddd", ec="white")
        ax.plot(GRID, mix_pdf(GRID, name), color="k", lw=1.0, ls=":", label="truth")
        for key, col, lbl, ls in [("mle", BLUE, "MLE", "-"), ("sm", RED, "score matching", "-"),
                                   ("dsm0.5", ORANGE, r"DSM $\sigma=0.5$", "-"), ("nce3", GREEN, r"NCE, $\mathcal{N}(0,3^2)$", "--"),
                                   ("nce0.3", MAGENTA, r"NCE, $\mathcal{N}(0,0.3^2)$", "--")]:
            ax.plot(GRID, poly_density(res[key])[0], color=col, lw=1.3, ls=ls, label=lbl)
        ax.set_xlim(-4, 4)
        ax.set_ylim(0, 0.55)
        ax.set_xlabel("$x$")
        if lab == "a":
            ax.set_ylabel("density")
            ax.legend(frameon=False, fontsize=7.5, loc="upper left")
        panel_label(ax, lab, "upper right")
    savefig(fig, CH, "figx24-b", out)


# =========================================================================== RBM small enough to enumerate
class RBMEnum:
    """A binary restricted Boltzmann machine with V visible and H hidden units, E(x, z) = -x'Wz - b'x - c'z,
    small enough to enumerate every state. All expectations are exact, so the learning curves of exact
    maximum likelihood, contrastive divergence (k Gibbs steps, expected update) and persistent
    contrastive divergence are free of sampling noise (Sections 24.2.2.1, 24.2.2.2)."""

    def __init__(self, V, H, W, b, c):
        self.V, self.H = V, H
        self.VS = np.array(list(itertools.product([0, 1], repeat=V)), float)
        self.HS = np.array(list(itertools.product([0, 1], repeat=H)), float)
        self.W, self.b, self.c = np.array(W, float), np.array(b, float), np.array(c, float)

    def log_unnorm_v(self):
        """log of the unnormalised marginal over visible states, -F(x) with the free energy F."""
        return self.VS @ self.b + softplus(self.VS @ self.W + self.c).sum(1)

    def pv(self):
        l = self.log_unnorm_v()
        p = np.exp(l - l.max())
        return p / p.sum()

    def joint(self):
        """p(x, z) as an array (2^V, 2^H), from the energy."""
        L = self.VS @ self.W @ self.HS.T + (self.VS @ self.b)[:, None] + (self.HS @ self.c)[None, :]
        L = L - L.max()
        P = np.exp(L)
        return P / P.sum()

    def kernel(self):
        """Transition matrix of one block Gibbs sweep x -> z -> x' on the visible states."""
        Ph = sigm(self.VS @ self.W + self.c)
        ph = np.prod(np.where(self.HS[None] == 1, Ph[:, None, :], 1 - Ph[:, None, :]), axis=2)
        Pv = sigm(self.HS @ self.W.T + self.b)
        pvh = np.prod(np.where(self.VS[None] == 1, Pv[:, None, :], 1 - Pv[:, None, :]), axis=2)
        return ph @ pvh

    def stats(self, p):
        """(E[x mu(x)'], E[x], E[mu(x)]) with x ~ p over visible states and mu(x) = E[z | x] = sigmoid(W'x + c)."""
        mu = sigm(self.VS @ self.W + self.c)
        return (self.VS * p[:, None]).T @ mu, (self.VS * p[:, None]).sum(0), (mu * p[:, None]).sum(0)

    def loglik_grad(self, pd):
        """Exact gradient of E_pD[log p(x)] with respect to (W, b, c): clamped minus unclamped expectations (24.21)."""
        gp, gn = self.stats(pd), self.stats(self.pv())
        return gp[0] - gn[0], gp[1] - gn[1], gp[2] - gn[2]

    def step(self, g, lr):
        self.W = self.W + lr * g[0]
        self.b = self.b + lr * g[1]
        self.c = self.c + lr * g[2]


def rbm_data_dist(flip=0.03):
    """Distribution on the 64 states of six binary pixels: three prototypes with independent bit flips."""
    VS = np.array(list(itertools.product([0, 1], repeat=6)), float)
    protos = np.array([[1, 1, 1, 0, 0, 0], [0, 0, 0, 1, 1, 1], [1, 0, 1, 0, 1, 0]], float)
    pd = np.zeros(64)
    for w, pr in zip([0.4, 0.35, 0.25], protos):
        d = np.abs(VS - pr).sum(1)
        pd += w * flip ** d * (1 - flip) ** (6 - d)
    return pd / pd.sum()


def rbm_init(seed=0, V=6, H=3):
    r = np.random.default_rng(seed)
    return 0.1 * r.standard_normal((V, H)), np.zeros(V), np.zeros(H)


def kl_disc(p, q):
    m = p > 0
    return float(np.sum(p[m] * np.log(p[m] / q[m])))


def rbm_train(method, pd, steps=6000, lr=0.2, k=1, seed=0, record=100):
    """Train the enumerable RBM; method 'ml' (exact gradient), 'cd' (k Gibbs steps from the data
    distribution, exact expectation) or 'pcd' (a persistent distribution advanced k steps per update).
    Returns the model and the list of (step, KL(pD || p_theta))."""
    W, b, c = rbm_init(seed)
    m = RBMEnum(6, 3, W, b, c)
    q = np.full(64, 1.0 / 64)
    hist = []
    for t in range(steps + 1):
        if t % record == 0:
            hist.append((t, kl_disc(pd, m.pv())))
        if t == steps:
            break
        K = m.kernel()
        if method == "ml":
            neg = m.pv()
        elif method == "cd":
            neg = pd.copy()
            for _ in range(k):
                neg = neg @ K
        else:
            for _ in range(k):
                q = q @ K
            neg = q
        gp, gn = m.stats(pd), m.stats(neg)
        m.step((gp[0] - gn[0], gp[1] - gn[1], gp[2] - gn[2]), lr)
    return m, hist


def fig_x24_c(out=None):
    """Own figure: exact maximum likelihood, CD-k and persistent CD on an RBM small enough to enumerate."""
    pd = rbm_data_dist()
    runs = {}
    for lab, meth, k in [("exact ML", "ml", 1), ("CD-1", "cd", 1), ("CD-3", "cd", 3), ("CD-20", "cd", 20), ("PCD-1", "pcd", 1)]:
        runs[lab] = rbm_train(meth, pd, k=k)
    fig, axes = panel_grid(1, 2, size=(3.5, 2.7))
    ax = axes[0, 0]
    cols = {"exact ML": "k", "CD-1": RED, "CD-3": ORANGE, "CD-20": GREEN, "PCD-1": BLUE}
    for lab, (m, h) in runs.items():
        h = np.array(h)
        ax.plot(h[1:, 0], h[1:, 1], color=cols[lab], lw=1.4, ls="--" if lab == "PCD-1" else "-", label=lab)
    ax.set_yscale("log")
    ax.set_xlabel("update")
    ax.set_ylabel(r"$D_{\mathrm{KL}}(p_{\mathcal{D}}\,\|\,p_\theta)$ (nats)")
    ax.legend(frameon=False, fontsize=8)
    panel_label(ax, "a", "upper right")
    ax = axes[0, 1]
    order = np.argsort(-pd)[:10]
    w = 0.27
    ax.bar(np.arange(10) - w, pd[order], w, color="#bbbbbb", label="data")
    ax.bar(np.arange(10), runs["exact ML"][0].pv()[order], w, color="k", label="ML")
    ax.bar(np.arange(10) + w, runs["CD-1"][0].pv()[order], w, color=RED, label="CD-1")
    ax.set_xticks(np.arange(10))
    ax.set_xticklabels(["".join(str(int(v)) for v in runs["ML" if False else "exact ML"][0].VS[i]) for i in order], rotation=90, fontsize=7)
    ax.set_ylabel("probability")
    ax.legend(frameon=False, fontsize=8)
    panel_label(ax, "b", "upper right")
    savefig(fig, CH, "figx24-c", out)


# =========================================================================== Algorithm 24.1 on the 8 x 8 digits
def train_rbm_digits(method="cd", n_hidden=64, epochs=80, lr=0.05, batch=50, seed=0):
    """Algorithm 24.1 (CD-1) and persistent CD (Algorithm 24.2, one chain per minibatch row) for a binary RBM on the
    8 x 8 digits (pixel intensity / 16 used as the probability that the visible unit is on).
    Returns (W, b, c, per-epoch reconstruction error)."""
    from sklearn.datasets import load_digits
    X = load_digits().data / 16.0
    rng = np.random.default_rng(seed)
    N, D = X.shape
    W = 0.01 * rng.standard_normal((D, n_hidden))
    b = np.zeros(D)
    c = np.zeros(n_hidden)
    chain = (rng.random((batch, D)) < 0.5).astype(float)
    err = []
    for ep in range(epochs):
        perm = rng.permutation(N)
        e = 0.0
        for i in range(0, N - batch + 1, batch):
            x = X[perm[i:i + batch]]
            mu = sigm(x @ W + c)
            z = (rng.random(mu.shape) < mu).astype(float)
            start = x if method == "cd" else chain
            zs = z if method == "cd" else (rng.random(mu.shape) < sigm(chain @ W + c)).astype(float)
            xp = (rng.random(x.shape) < sigm(zs @ W.T + b)).astype(float)
            mup = sigm(xp @ W + c)
            if method == "pcd":
                chain = xp
            W += lr * (x.T @ mu - xp.T @ mup) / len(x)
            b += lr * (x - xp).mean(0)
            c += lr * (mu - mup).mean(0)
            e += ((x - sigm(z @ W.T + b)) ** 2).sum()
        err.append(e / N)
    return W, b, c, np.array(err)


def rbm_fantasy(W, b, c, n=16, steps=600, seed=1):
    rng = np.random.default_rng(seed)
    x = (rng.random((n, W.shape[0])) < 0.5).astype(float)
    for _ in range(steps):
        z = (rng.random((n, W.shape[1])) < sigm(x @ W + c)).astype(float)
        p = sigm(z @ W.T + b)
        x = (rng.random(p.shape) < p).astype(float)
    return p


def _tile(M, nrow, ncol, side=8, fill=0.0):
    T = np.full((nrow * (side + 1), ncol * (side + 1)), fill)
    for i in range(nrow * ncol):
        r, c = divmod(i, ncol)
        T[r * (side + 1):r * (side + 1) + side, c * (side + 1):c * (side + 1) + side] = M[i].reshape(side, side)
    return T


def fig_x24_d(out=None):
    """Own figure: weights and fantasy particles of an RBM on 8 x 8 digits trained with CD-1 and with persistent chains."""
    fig, axes = panel_grid(2, 2, size=(3.0, 3.0))
    for row, meth, name in [(0, "cd", "CD-1"), (1, "pcd", "persistent CD")]:
        W, b, c, err = train_rbm_digits(meth)
        ax = axes[row, 0]
        ax.imshow(_tile(W.T[:25] / np.abs(W.T[:25]).max(1, keepdims=True), 5, 5), cmap="RdBu_r", vmin=-1, vmax=1)
        ax.set_title("%s: 25 of 64 filters (each scaled to its own maximum)" % name, fontsize=9)
        ax.axis("off")
        ax = axes[row, 1]
        fp = rbm_fantasy(W, b, c, 16)
        ax.imshow(_tile(fp, 4, 4), cmap="gray_r", vmin=0, vmax=1)
        ax.set_title("fantasy particles after 600 Gibbs sweeps", fontsize=9)
        ax.axis("off")
    savefig(fig, CH, "figx24-d", out)


# =========================================================================== Section 24.3: DSM and SSM variance on a Gaussian
def dsm_optimal_precision(s2, sigma):
    """Precision of the Gaussian EBM minimising denoising score matching when the data are N(0, s2): the
    precision of the noisy data, 1 / (s2 + sigma^2) (gap 24.8)."""
    return 1.0 / (s2 + sigma ** 2)


def dsm_grad_std(s2, sigma, a, n=200000, seed=0):
    """Standard deviation of one per-sample DSM gradient d/da of (1/2)(s(x~) - (x - x~)/sigma^2)^2 for the Gaussian EBM
    with precision a: s(x~) = -a x~, x ~ N(0, s2), x~ = x + sigma z."""
    r = np.random.default_rng(seed)
    x = np.sqrt(s2) * r.standard_normal(n)
    z = r.standard_normal(n)
    xt = x + sigma * z
    s = -a * xt
    target = -z / sigma                  # (x - x~)/sigma^2
    g = (s - target) * (-xt)             # d/da of (1/2)(s - target)^2
    return float(g.std())


def fig_x24_e(out=None):
    """Own figure: bias and variance of denoising score matching as the noise scale shrinks."""
    sig = np.logspace(-2, 0.3, 14)
    fig, axes = panel_grid(1, 2, size=(3.4, 2.7))
    ax = axes[0, 0]
    ax.plot(sig, dsm_optimal_precision(1.0, sig), color=RED, lw=1.5, label="DSM optimum")
    ax.axhline(1.0, color="k", ls=":", lw=0.9, label="true precision")
    ax.set_xscale("log")
    ax.set_xlabel(r"noise scale $\sigma$")
    ax.set_ylabel("precision of the fitted Gaussian")
    ax.legend(frameon=False)
    panel_label(ax, "a", "lower left")
    ax = axes[0, 1]
    ds = np.array([dsm_grad_std(1.0, s, 1.0) for s in sig])
    ax.loglog(sig, ds, "o", color=BLUE, ms=4, label="simulated")
    ax.loglog(sig, ds[0] * sig[0] / sig, color=GREY, ls="--", label=r"$\propto 1/\sigma$")
    ax.set_xlabel(r"noise scale $\sigma$")
    ax.set_ylabel("std of the per-sample gradient")
    ax.legend(frameon=False)
    panel_label(ax, "b", "lower left")
    savefig(fig, CH, "figx24-e", out)


# =========================================================================== Section 24.4: NCE and the noise distribution
def nce_gauss_fit(x, s_noise, nu=1.0, rng=None, n_noise=None):
    """NCE for the Gaussian EBM E = a x^2 / 2 with a learned c = -log Z, noise N(0, s_noise^2), by Newton.
    Returns (a, c)."""
    rng = rng or np.random.default_rng(0)
    n_noise = n_noise or int(len(x) * nu)
    y = s_noise * rng.standard_normal(n_noise)
    lpn = lambda t: -0.5 * (t / s_noise) ** 2 - np.log(s_noise * np.sqrt(2 * np.pi))
    Fd = np.stack([-0.5 * x ** 2, np.ones_like(x)], 1)
    Fn = np.stack([-0.5 * y ** 2, np.ones_like(y)], 1)
    od = np.log(nu) - lpn(x)
    on = np.log(nu) - lpn(y)
    th = np.array([0.5, -0.9])
    q1, q0 = nu / (1 + nu), 1 / (1 + nu)

    def obj(t):
        return q1 * np.mean(softplus(-(Fd @ t + od))) + q0 * np.mean(softplus(Fn @ t + on))

    for _ in range(60):
        sd, sn = sigm(Fd @ th + od), sigm(Fn @ th + on)
        g = q1 * (-(1 - sd)[:, None] * Fd).mean(0) + q0 * (sn[:, None] * Fn).mean(0)
        H = q1 * ((sd * (1 - sd))[:, None] * Fd).T @ Fd / len(x) + q0 * ((sn * (1 - sn))[:, None] * Fn).T @ Fn / len(y)
        st = np.linalg.solve(H + 1e-10 * np.eye(2), g)
        f0, a = obj(th), 1.0
        while obj(th - a * st) > f0 + 1e-15 and a > 1e-10:
            a /= 2
        th = th - a * st
        if np.abs(a * st).max() < 1e-11:
            break
    return th[0], th[1]


def nce_logz_rmse(s_list, n=500, reps=120, nu=1.0, seed=3):
    """Root-mean-square error of the NCE estimate of log Z for N(0,1) data, against the noise scale."""
    rng = np.random.default_rng(seed)
    out = []
    truth = 0.5 * np.log(2 * np.pi)
    for s in s_list:
        e = []
        for _ in range(reps):
            x = rng.standard_normal(n)
            a, c = nce_gauss_fit(x, s, nu, rng)
            e.append(-c - truth)
        out.append(np.sqrt(np.mean(np.square(e))))
    return np.array(out)


def fig_x24_f(out=None):
    """Own figure: NCE error in log Z against the width of the noise distribution, for the number of noise samples nu * n."""
    s = np.array([0.5, 0.8, 1.2, 1.8, 2.5, 3.5, 5.0, 8.0, 12.0])
    fig, ax = plt.subplots(figsize=(3.6, 2.7))
    for nu, col in [(1.0, BLUE), (4.0, GREEN)]:
        ax.plot(s, nce_logz_rmse(s, nu=nu), "o-", color=col, ms=4, label=r"$\nu=%g$" % nu)
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_xlabel(r"noise standard deviation (data: $\mathcal{N}(0,1)$)")
    ax.set_ylabel(r"RMSE of the estimate of $\log Z$")
    ax.legend(frameon=False)
    savefig(fig, CH, "figx24-f", out)


# =========================================================================== Neural energy on a swiss roll (24.3.2, 24.3.3)
def swiss_roll(n=2000, seed=0):
    from sklearn.datasets import make_swiss_roll
    X, _ = make_swiss_roll(n, noise=0.3, random_state=seed)
    X = X[:, [0, 2]]
    return ((X - X.mean(0)) / X.std(0)).astype(np.float32)


def train_neural_ebm(method, X, steps=4000, sigma=0.15, seed=0, lr=2e-3):
    """Energy network 2-128-128-1 with softplus units, trained by denoising ('dsm') or sliced ('ssm', Gaussian
    projections, exact double backward) score matching. Returns the network."""
    import torch
    torch.manual_seed(seed)
    net = torch.nn.Sequential(torch.nn.Linear(2, 128), torch.nn.Softplus(), torch.nn.Linear(128, 128), torch.nn.Softplus(), torch.nn.Linear(128, 1))
    opt = torch.optim.Adam(net.parameters(), lr=lr)
    Xt = torch.tensor(X)
    g = torch.Generator().manual_seed(seed)
    for it in range(steps):
        idx = torch.randint(0, len(Xt), (256,), generator=g)
        x = Xt[idx]
        if method == "dsm":
            xt = (x + sigma * torch.randn(x.shape, generator=g)).requires_grad_(True)
            e = net(xt).sum()
            s = -torch.autograd.grad(e, xt, create_graph=True)[0]
            loss = 0.5 * ((s - (x - xt) / sigma ** 2) ** 2).sum(1).mean() * sigma ** 2     # scaled by sigma^2: same minimiser
        else:
            xr = x.clone().requires_grad_(True)
            e = net(xr).sum()
            s = -torch.autograd.grad(e, xr, create_graph=True)[0]
            v = torch.randn(x.shape, generator=g)
            sv = (s * v).sum()
            Jv = torch.autograd.grad(sv, xr, create_graph=True)[0]
            loss = (0.5 * (s * v).sum(1) ** 2 + (Jv * v).sum(1)).mean()
        opt.zero_grad()
        loss.backward()
        opt.step()
    return net


def neural_langevin(net, n=1500, eps=0.07, K=600, seed=3, noise_scale=1.0):
    import torch
    g = torch.Generator().manual_seed(seed)
    x = (3 * torch.randn((n, 2), generator=g)).requires_grad_(True)
    for _ in range(K):
        e = net(x).sum()
        gr = torch.autograd.grad(e, x)[0]
        x = (x - 0.5 * eps ** 2 * gr + noise_scale * eps * torch.randn(x.shape, generator=g)).detach().requires_grad_(True)
    return x.detach().numpy()


def fig_x24_g(out=None):
    """Own figure: a neural energy fitted by denoising and sliced score matching on a swiss roll, and Langevin samples."""
    import torch
    X = swiss_roll()
    fig, axes = panel_grid(1, 3, size=(2.7, 2.7))
    ax = axes[0, 0]
    ax.plot(X[:, 0], X[:, 1], ".", ms=1.5, color=GREY)
    ax.set_title("data", fontsize=9)
    g = np.linspace(-2.6, 2.6, 80)
    G1, G2 = np.meshgrid(g, g)
    P = torch.tensor(np.stack([G1.ravel(), G2.ravel()], 1).astype(np.float32))
    for ax, meth, name in [(axes[0, 1], "dsm", r"denoising SM ($\sigma=0.15$)"), (axes[0, 2], "ssm", "sliced SM")]:
        net = train_neural_ebm(meth, X)
        with torch.no_grad():
            E = net(P).numpy().reshape(80, 80)
        ax.contourf(G1, G2, E, levels=20, cmap="Blues_r", alpha=0.8)
        S = neural_langevin(net)
        ax.plot(S[:, 0], S[:, 1], ".", ms=1.6, color=RED, mec="none", alpha=0.8)
        ax.set_title(name, fontsize=9)
    for ax in axes[0]:
        ax.set_xlim(-2.6, 2.6)
        ax.set_ylim(-2.6, 2.6)
        ax.set_xticks([])
        ax.set_yticks([])
    savefig(fig, CH, "figx24-g", out)


# =========================================================================== data for the demos
def export_demo_data():
    import json
    os.makedirs(DATA_OUT, exist_ok=True)
    # == the estimator demo
    ds = {}
    for name in DATASETS:
        x = mix_sample(name)
        d = DATASETS[name]
        ref = {}
        th = fit_mle_poly(x)
        ref["mle"] = dict(theta=th.tolist(), kl=kl_to_truth(name, th), logz=float(poly_density(th)[1]))
        th = fit_sm_poly(x)
        ref["sm"] = dict(theta=th.tolist(), kl=kl_to_truth(name, th))
        for sg in (0.2, 0.5, 1.0):
            th = fit_sm_poly(x, sg)
            ref["dsm%g" % sg] = dict(theta=th.tolist(), kl=kl_to_truth(name, th))
        for s in (0.3, 1.0, 3.0, 10.0):
            th, c = fit_nce_poly(x, 0.0, s)
            ref["nce%g" % s] = dict(theta=th.tolist(), c=float(c), kl=kl_to_truth(name, th), logz=float(poly_density(th)[1]))
        ds[name] = dict(x=[round(float(v), 6) for v in x], w=d["w"], m=d["m"], s=d["s"], ref=ref)
    with open(os.path.join(DATA_OUT, "ch24-estimators.json"), "w") as f:
        json.dump(ds, f, separators=(",", ":"))
    # == the product of experts demo: reference numbers
    poe = dict(centres=POE_CENTRES.tolist(), s=POE_S, kappa=POE_KAPPA,
               right_mass_AB=poe_right_mass(("A", "B")), right_mass_A=poe_right_mass(("A",)),
               ula_var={"%g" % e: ula_gaussian_variance(1.0, e) for e in (0.3, 0.6, 1.0, 1.5, 1.8)})
    with open(os.path.join(DATA_OUT, "ch24-poe.json"), "w") as f:
        json.dump(poe, f, separators=(",", ":"))
    # == the RBM demo
    pd = rbm_data_dist()
    W, b, c = rbm_init(0)
    ref = {}
    for lab, meth, k in [("ml", "ml", 1), ("cd1", "cd", 1), ("cd3", "cd", 3), ("cd20", "cd", 20), ("pcd1", "pcd", 1)]:
        m, h = rbm_train(meth, pd, k=k)
        ref[lab] = dict(kl=[round(v[1], 8) for v in h], final=h[-1][1])
    rb = dict(pd=pd.tolist(), W=W.tolist(), b=b.tolist(), c=c.tolist(), lr=0.2, steps=6000, record=100, ref=ref)
    with open(os.path.join(DATA_OUT, "ch24-rbm.json"), "w") as f:
        json.dump(rb, f, separators=(",", ":"))
    print("wrote data/ch24-*.json")


def fig_data(out=None):
    """Not a figure: export the JSON data used by the demonstrations."""
    export_demo_data()


FIGURES = {
    "24.1": fig_24_1,
    "x24.a": fig_x24_a,
    "x24.b": fig_x24_b,
    "x24.c": fig_x24_c,
    "x24.d": fig_x24_d,
    "x24.e": fig_x24_e,
    "x24.f": fig_x24_f,
    "x24.g": fig_x24_g,
    "data": fig_data,
}

if __name__ == "__main__":
    common.run(FIGURES, CH)
