"""Bishop and Bishop (2024), Chapter 16: Continuous Latent Variables.

Reproduces the figures of Chapter 16 of C. M. Bishop and H. Bishop, Deep Learning: Foundations and
Concepts (Springer, 2024), pp. 495-531, written from scratch with numpy, scipy (image resampling
and special functions only) and matplotlib, and supplies the data and reference numbers of the three
browser demonstrations. Every algorithm is a short function meant to be read next to the lecture
notes (ch16-continuous-latent-variables.html):

* Section 16.1, principal component analysis: the eigen-decomposition of the sample covariance
  (pca), the N x N "snapshot" route for N < D (gram_pca, (16.26)-(16.30)), reconstructions (16.20),
  and whitening (16.24).
* Section 16.2, probabilistic latent variables: the closed-form maximum likelihood probabilistic PCA
  (ppca_ml, (16.46)-(16.47)), its log likelihood (16.44) with the O(M^3) inverse (16.41), the
  posterior (16.43); factor analysis (fa_em, fa_loglik); independent component analysis by maximum
  likelihood with the sech prior (16.56) or a light-tailed prior, natural-gradient ascent and the
  likelihood along rotations of whitened data (ica_*); the Kalman filter for a linear dynamical
  system (kalman_filter), checked against the joint Gaussian likelihood.
* Section 16.3, the evidence lower bound: EM for probabilistic PCA (em_ppca, (16.66)-(16.69)), EM
  for PCA in the zero-noise limit (em_pca_step, (16.70)-(16.71)), EM for factor analysis
  ((16.72)-(16.76)); the ELBO of a Gaussian q for a nonlinear model (elbo_gauss).
* Section 16.4, nonlinear latent variable models: the circle model of Figure 16.13 and its marginal
  density by quadrature, the simple Monte Carlo likelihood estimate (16.83) and its variance,
  importance sampling, a small tanh network g(z, w) with a one-dimensional latent space trained by
  maximising (16.83) with Adam (nlvm_*), dequantisation (16.4.3).
* JSRandom: the seeded generator DK.rng of the browser demos (mulberry32 uniforms, Box-Muller
  normals). The ICA and nonlinear-latent demos draw their data and initial weights from it, so
  ica-unmixing-demo.html and nonlinear-latent-demo.html reproduce ica_natgrad and nlvm_train number
  for number; eigen-digits-demo.html uses the eigenvectors exported by export_eigendigits_json.

Figures reproduced: 16.1, 16.2, 16.3, 16.4, 16.5, 16.6, 16.7, 16.10, 16.11, 16.13, 16.14 and 16.15.
Figure 16.1 is built from an MNIST digit (the book's caption uses a larger image; here a 28 x 28
MNIST digit is enlarged to 64 x 64 and placed in a 100 x 100 field). Figures 16.2, 16.7, 16.11 and
16.15 are schematic in the book and are computed here from explicit data, densities or a network
with fixed random weights. Figure 16.14 in the book reprints three images from Doersch (2016); here
the same phenomenon is reproduced with an MNIST '2' (a removed piece of stroke against a half-pixel
shift), and our own distances are reported.
Skipped: Figures 16.8, 16.9 and 16.12 (hand-drawn graphical models with no data).

Supplementary figures (our own, numbered 16.S1-16.S7 in the order they appear in the notes):
16.S1 the PCA spectrum of the translated-and-rotated digit images of Figure 16.1: three degrees of
freedom, but many linear components; 16.S2 independent component analysis of two sources, whitening,
and the likelihood along rotations for heavy-tailed, light-tailed and Gaussian sources; 16.S3 the
Kalman filter for a one-dimensional linear dynamical system; 16.S4 EM for probabilistic PCA on the
MNIST threes and EM for PCA as subspace iteration; 16.S5 the cost of the Monte Carlo likelihood
(16.83) as the noise shrinks, and importance sampling; 16.S6 the evidence lower bound for a
Gaussian q and a bimodal posterior; 16.S7 a nonlinear latent variable model trained by maximum
likelihood, against probabilistic PCA.

Run `python3 ch16_continuous_latent_variables.py --list`, `python3 ch16_continuous_latent_variables.py`
(all figures, about a minute on one CPU thread) or `... --fig 16.5 16.S2`. Files go to ../img/ch16/.
"""
import base64
import json
import math
import os

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import numpy as np  # noqa: E402
import matplotlib.pyplot as plt  # noqa: E402
from matplotlib.patches import Circle, Ellipse  # noqa: E402
from scipy import ndimage, special  # noqa: E402

import common  # noqa: E402
from common import savefig, run  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
WEB_DATA = os.path.normpath(os.path.join(HERE, "..", "data"))
CACHE_DIR = os.path.join(HERE, "cache")
CH = 16
M32 = 0xFFFFFFFF
LN2PI = math.log(2.0 * math.pi)


# =========================================================================== the demos' generator
def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js).

    Uniforms come from the mulberry32 generator and normals from the Box-Muller transform, the
    second normal of each pair being cached, exactly as in JavaScript.
    """

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def _pos(self):
        x = 0.0
        while x == 0.0:
            x = self.u()
        return x

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        r = math.sqrt(-2.0 * math.log(self._pos()))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


# =========================================================================== 16.1 PCA
def threes(n=6000, scale=255.0, train=True):
    """The first n MNIST threes as an (n, 784) array; scale=255 gives grey levels 0..255 (Figure 16.3)."""
    X, _ = common.mnist(train=train, digits=(3,))
    return X[:n] * scale


def pca(X):
    """Mean, eigenvalues (decreasing) and eigenvectors (columns) of the covariance (16.3), divisor N."""
    xbar = X.mean(0)
    Xc = X - xbar
    S = Xc.T @ Xc / X.shape[0]
    lam, U = np.linalg.eigh(S)
    order = np.argsort(lam)[::-1]
    return xbar, np.maximum(lam[order], 0.0), U[:, order]


def gram_pca(X, M):
    """PCA through the N x N matrix (16.28) and the normalisation (16.30): returns xbar, lambda, U (D x M)."""
    xbar = X.mean(0)
    Xc = X - xbar
    N = X.shape[0]
    K = Xc @ Xc.T / N
    lam, V = np.linalg.eigh(K)
    order = np.argsort(lam)[::-1][:M]
    lam, V = lam[order], V[:, order]
    U = Xc.T @ V / np.sqrt(N * np.maximum(lam, 1e-300))
    return xbar, lam, U


def reconstruct(x, xbar, U, M):
    """The M-term PCA approximation (16.20)."""
    Um = U[:, :M]
    return xbar + Um @ (Um.T @ (x - xbar))


def whiten(X):
    """Whitening (16.24): rows y_n = L^{-1/2} U^T (x_n - xbar); returns Y, xbar, U, lam."""
    xbar, lam, U = pca(X)
    Y = (X - xbar) @ U / np.sqrt(lam)
    return Y, xbar, U, lam


# =========================================================================== 16.2 probabilistic PCA
def ppca_ml(S, M):
    """Closed-form maximum likelihood (16.46)-(16.47) with R = I: W (D x M) and sigma^2."""
    lam, U = np.linalg.eigh(S)
    order = np.argsort(lam)[::-1]
    lam, U = lam[order], U[:, order]
    D = S.shape[0]
    s2 = lam[M:].sum() / (D - M)
    W = U[:, :M] * np.sqrt(np.maximum(lam[:M] - s2, 0.0))
    return W, s2


def ppca_cov(W, s2):
    return W @ W.T + s2 * np.eye(W.shape[0])


def ppca_cinv(W, s2):
    """C^{-1} through the M x M matrix M = W^T W + s2 I, (16.41)-(16.42)."""
    Mm = W.T @ W + s2 * np.eye(W.shape[1])
    return (np.eye(W.shape[0]) - W @ np.linalg.solve(Mm, W.T)) / s2


def ppca_loglik_S(S, W, s2, N):
    """(16.45): ln p(X) = -N/2 {D ln 2pi + ln|C| + Tr(C^{-1} S)} at mu = xbar."""
    D, M = W.shape
    Mm = W.T @ W + s2 * np.eye(M)
    logdetC = (D - M) * math.log(s2) + np.linalg.slogdet(Mm)[1]      # |C| = s2^{D-M} |M|
    tr = (np.trace(S) - np.sum(np.linalg.solve(Mm, W.T @ S) * W.T)) / s2     # Tr(C^{-1} S) via (16.41)
    return -0.5 * N * (D * LN2PI + logdetC + tr)


def ppca_posterior(x, mu, W, s2):
    """(16.43): mean M^{-1} W^T (x - mu) and covariance s2 M^{-1}."""
    Mm = W.T @ W + s2 * np.eye(W.shape[1])
    return np.linalg.solve(Mm, W.T @ (x - mu)), s2 * np.linalg.inv(Mm)


def em_ppca(X, M, iters, W0=None, s20=1.0, seed=0, track=True):
    """EM for probabilistic PCA, (16.66)-(16.69); returns W, sigma^2 and the log likelihood per iteration."""
    N, D = X.shape
    xbar = X.mean(0)
    Xc = X - xbar
    S = Xc.T @ Xc / N if track else None
    rng = np.random.default_rng(seed)
    W = rng.standard_normal((D, M)) if W0 is None else W0.copy()
    s2 = s20
    hist = []
    for _ in range(iters):
        Mm = W.T @ W + s2 * np.eye(M)
        Minv = np.linalg.inv(Mm)
        Ez = Xc @ W @ Minv                         # rows E[z_n]^T, (16.66)
        sumEzz = N * s2 * Minv + Ez.T @ Ez         # sum_n E[z_n z_n^T], (16.67)
        Wn = (Xc.T @ Ez) @ np.linalg.inv(sumEzz)   # (16.68)
        s2 = (np.sum(Xc ** 2) - 2 * np.sum((Ez @ Wn.T) * Xc) + np.trace(sumEzz @ Wn.T @ Wn)) / (N * D)   # (16.69)
        W = Wn
        if track:
            hist.append(ppca_loglik_S(S, W, s2, N))
    return W, s2, np.array(hist)


def ppca_loglik_missing(X, mask, W, mu, s2):
    """Observed-data log likelihood: sum_n ln N(x_n^o | mu_o, W_o W_o^T + s2 I); mask True = observed."""
    ll = 0.0
    for x, o in zip(X, mask):
        Wo = W[o]
        C = Wo @ Wo.T + s2 * np.eye(o.sum())
        d = x[o] - mu[o]
        ll += -0.5 * (o.sum() * LN2PI + np.linalg.slogdet(C)[1] + d @ np.linalg.solve(C, d))
    return ll


def em_ppca_missing(X, mask, M, iters, W0, mu0, s20):
    """EM for probabilistic PCA with values missing at random (Exercise 16.22).

    Latent: z_n and the missing x_n^m. E step: p(z_n | x_n^o) has M_n = W_o^T W_o + s2 I, mean
    M_n^{-1} W_o^T (x_o - mu_o), covariance s2 M_n^{-1}; the missing entries enter through
    E[x_ni z~^T] and E[x_ni^2] with z~ = (z, 1). M step: each row (w_i, mu_i) by least squares in z~,
    then sigma^2. With no missing values and mu0 = xbar it is exactly (16.66)-(16.69)."""
    N, D = X.shape
    W, mu, s2 = W0.copy(), mu0.copy(), s20
    hist = []
    for _ in range(iters):
        Szz = np.zeros((M + 1, M + 1))
        Sxz = np.zeros((D, M + 1))
        Sxx = np.zeros(D)
        for x, o in zip(X, mask):
            Wo = W[o]
            Mn = Wo.T @ Wo + s2 * np.eye(M)
            Mi = np.linalg.inv(Mn)
            Ez = Mi @ Wo.T @ (x[o] - mu[o])
            Ezz = s2 * Mi + np.outer(Ez, Ez)
            Et = np.append(Ez, 1.0)
            Ett = np.zeros((M + 1, M + 1))
            Ett[:M, :M] = Ezz
            Ett[:M, M] = Ett[M, :M] = Ez
            Ett[M, M] = 1.0
            Szz += Ett
            th = np.hstack([W, mu[:, None]])          # rows (w_i, mu_i) at the old parameters
            mis = ~o
            Sxz[o] += np.outer(x[o], Et)
            Sxx[o] += x[o] ** 2
            Sxz[mis] += th[mis] @ Ett
            Sxx[mis] += np.einsum("ij,jk,ik->i", th[mis], Ett, th[mis]) + s2
        Th = Sxz @ np.linalg.inv(Szz)
        s2 = float((Sxx.sum() - 2 * np.sum(Th * Sxz) + np.sum((Th @ Szz) * Th)) / (N * D))
        W, mu = Th[:, :M].copy(), Th[:, M].copy()
        hist.append(ppca_loglik_missing(X, mask, W, mu, s2))
    return W, mu, s2, np.array(hist)


def em_pca_step(Xt, W):
    """One cycle of EM for PCA (sigma^2 -> 0): E step (16.70) and M step (16.71). Xt is N x D centred."""
    Om = np.linalg.solve(W.T @ W, W.T @ Xt.T)           # M x N
    Wn = Xt.T @ Om.T @ np.linalg.inv(Om @ Om.T)
    return Wn, Om


def subspace_angle(A, B):
    """Largest principal angle (radians) between the column spaces of A and B."""
    Qa, _ = np.linalg.qr(A)
    Qb, _ = np.linalg.qr(B)
    s = np.linalg.svd(Qa.T @ Qb, compute_uv=False)
    return float(np.arccos(np.clip(s.min(), -1, 1)))


# =========================================================================== factor analysis
def fa_loglik(X, W, Psi):
    N, D = X.shape
    C = W @ W.T + np.diag(Psi)
    xbar = X.mean(0)
    S = (X - xbar).T @ (X - xbar) / N
    return -0.5 * N * (D * LN2PI + np.linalg.slogdet(C)[1] + np.trace(np.linalg.solve(C, S)))


def fa_em(X, M, iters, W0=None, Psi0=None, seed=0):
    """EM for factor analysis, (16.72)-(16.76)."""
    N, D = X.shape
    xbar = X.mean(0)
    Xc = X - xbar
    S = Xc.T @ Xc / N
    rng = np.random.default_rng(seed)
    W = rng.standard_normal((D, M)) if W0 is None else W0.copy()
    Psi = np.diag(S).copy() if Psi0 is None else Psi0.copy()
    hist = []
    for _ in range(iters):
        G = np.linalg.inv(np.eye(M) + (W.T / Psi) @ W)            # (16.74)
        Ez = Xc @ (G @ (W.T / Psi)).T                              # rows E[z_n]^T, (16.72)
        sumEzz = N * G + Ez.T @ Ez                                 # (16.73)
        W = (Xc.T @ Ez) @ np.linalg.inv(sumEzz)                    # (16.75)
        Psi = np.diag(S - W @ (Ez.T @ Xc) / N).copy()              # (16.76)
        hist.append(fa_loglik(X, W, Psi))
    return W, Psi, np.array(hist)


# =========================================================================== 16.2.5 ICA
def sech_sample(u):
    """Inverse CDF of p(z) = 1/(pi cosh z), (16.56): F(z) = (2/pi) arctan(e^z)."""
    return math.log(math.tan(math.pi * u / 2.0))


ICA_PRIORS = ("sech", "light")
LOGC_LIGHT = math.log(4.0 ** 0.25 * math.gamma(0.25) / 2.0)   # ln of the integral of exp(-z^4/4)


def ica_logp(z, prior="sech"):
    """Elementwise ln p(z): sech prior (16.56) or the light-tailed exp(-z^4/4)/c."""
    z = np.asarray(z, dtype=float)
    if prior == "sech":
        a = np.abs(z)
        return -math.log(math.pi) - a - np.log1p(np.exp(-2 * a)) + math.log(2.0)   # ln(2/(pi(e^z+e^-z)))
    return -z ** 4 / 4.0 - LOGC_LIGHT


def ica_phi(z, prior="sech"):
    """phi(z) = -d ln p / dz."""
    return np.tanh(z) if prior == "sech" else z ** 3


def ica_sources(kind, N, seed):
    """Two independent unit-variance sources drawn with the demos' generator (rows are time steps)."""
    rng = JSRandom(seed)
    S = np.zeros((N, 2))
    for n in range(N):
        for j in range(2):
            if kind == "sech":
                S[n, j] = sech_sample(rng._pos()) * (2.0 / math.pi)        # var of sech law is pi^2/4
            elif kind == "uniform":
                S[n, j] = math.sqrt(3.0) * (2.0 * rng.u() - 1.0)
            elif kind == "gauss":
                S[n, j] = rng.norm()
            elif kind == "signals":
                if j == 0:
                    S[n, j] = math.sqrt(2.0) * math.sin(2.0 * math.pi * n / 37.0)
                else:
                    f = n / 23.0
                    S[n, j] = math.sqrt(3.0) * (2.0 * (f - math.floor(f)) - 1.0)
    return S


def ica_mixing(a1_deg, a2_deg, l1=1.0, l2=0.6):
    """A has columns l_j (cos a_j, sin a_j): the directions of the two sources in data space."""
    a1, a2 = math.radians(a1_deg), math.radians(a2_deg)
    return np.array([[l1 * math.cos(a1), l2 * math.cos(a2)], [l1 * math.sin(a1), l2 * math.sin(a2)]])


def whiten2(X):
    """Whitening of 2-D data by an explicit symmetric eigen-solve (the same closed form as the demo)."""
    N = X.shape[0]
    m = X.mean(0)
    Xc = X - m
    a = np.sum(Xc[:, 0] * Xc[:, 0]) / N
    b = np.sum(Xc[:, 0] * Xc[:, 1]) / N
    c = np.sum(Xc[:, 1] * Xc[:, 1]) / N
    tr, det = a + c, a * c - b * b
    disc = math.sqrt(max(tr * tr / 4 - det, 0.0))
    l1, l2 = tr / 2 + disc, tr / 2 - disc
    th = 0.5 * math.atan2(2 * b, a - c)
    u1 = np.array([math.cos(th), math.sin(th)])
    u2 = np.array([-math.sin(th), math.cos(th)])
    V = np.array([u1 / math.sqrt(l1), u2 / math.sqrt(l2)])      # rows: L^{-1/2} U^T
    return Xc @ V.T, V, m


def rot(t):
    return np.array([[math.cos(t), -math.sin(t)], [math.sin(t), math.cos(t)]])


def ica_avg_loglik(W, Y, prior="sech"):
    """Average log likelihood per point of whitened data Y under unmixing W: ln|det W| + mean sum_j ln p(z_j)."""
    Z = Y @ W.T
    return math.log(abs(np.linalg.det(W))) + float(np.sum(ica_logp(Z, prior)) / Y.shape[0])


def ica_natgrad(Y, W0, eta, steps, prior="sech"):
    """Natural-gradient ascent W <- W + eta (I - mean phi(z) z^T) W; returns W and the log likelihood path."""
    W = W0.copy()
    N = Y.shape[0]
    path = [ica_avg_loglik(W, Y, prior)]
    for _ in range(steps):
        Z = Y @ W.T
        G = np.eye(2) - ica_phi(Z, prior).T @ Z / N
        W = W + eta * G @ W
        path.append(ica_avg_loglik(W, Y, prior))
    return W, np.array(path)


def polar_angle(W):
    """Angle of the rotation factor in the polar decomposition W = R P (2 x 2, det W > 0)."""
    return math.atan2(W[1, 0] - W[0, 1], W[0, 0] + W[1, 1])


def ica_problem(kind="sech", N=500, seed=7, a1=20.0, a2=70.0):
    S = ica_sources(kind, N, seed)
    A = ica_mixing(a1, a2)
    X = S @ A.T
    Y, V, m = whiten2(X)
    return S, A, X, Y, V


def abs_corr(Z, S):
    Zc, Sc = Z - Z.mean(0), S - S.mean(0)
    C = (Zc.T @ Sc) / np.sqrt(np.outer(np.sum(Zc ** 2, 0), np.sum(Sc ** 2, 0)))
    return np.abs(C)


# =========================================================================== 16.2.6 Kalman filter
def lds_simulate(T, a, q, c, r, seed=0):
    rng = np.random.default_rng(seed)
    z = np.zeros(T)
    x = np.zeros(T)
    z[0] = rng.normal(0, math.sqrt(q / (1 - a * a)))
    for t in range(T):
        if t:
            z[t] = a * z[t - 1] + rng.normal(0, math.sqrt(q))
        x[t] = c * z[t] + rng.normal(0, math.sqrt(r))
    return z, x


def kalman_filter(x, a, q, c, r, m0, P0):
    """Scalar Kalman filter: filtered means, variances, gains and the log likelihood ln p(x_1..x_T)."""
    T = len(x)
    m, P, K = np.zeros(T), np.zeros(T), np.zeros(T)
    ll = 0.0
    mp, Pp = m0, P0
    for t in range(T):
        if t:
            mp, Pp = a * m[t - 1], a * a * P[t - 1] + q          # predict
        s = c * c * Pp + r                                       # innovation variance
        ll += -0.5 * (LN2PI + math.log(s) + (x[t] - c * mp) ** 2 / s)
        K[t] = Pp * c / s
        m[t] = mp + K[t] * (x[t] - c * mp)
        P[t] = (1 - K[t] * c) * Pp
    return m, P, K, ll


def lds_joint_loglik(x, a, q, c, r, m0, P0):
    """ln p(x_1..x_T) from the joint Gaussian of the whole sequence (for checking the filter)."""
    T = len(x)
    Cz = np.zeros((T, T))
    var = np.zeros(T)
    var[0] = P0
    for t in range(1, T):
        var[t] = a * a * var[t - 1] + q
    for s in range(T):
        for t in range(s, T):
            Cz[s, t] = Cz[t, s] = a ** (t - s) * var[s]
    mean = c * m0 * a ** np.arange(T)
    C = c * c * Cz + r * np.eye(T)
    d = x - mean
    return float(-0.5 * (T * LN2PI + np.linalg.slogdet(C)[1] + d @ np.linalg.solve(C, d)))


# =========================================================================== 16.4 nonlinear models
def g_circle(z):
    """The book's Figure 16.13 mapping g(z) = (sin z, cos z)."""
    z = np.asarray(z, dtype=float)
    return np.stack([np.sin(z), np.cos(z)], -1)


ZGRID = np.linspace(-6.0, 6.0, 4001)


def log_px_quad(x, g, sigma, zgrid=ZGRID):
    """ln p(x) = ln int N(x | g(z), sigma^2 I) N(z | 0, 1) dz by the trapezoid rule on a fine grid."""
    G = g(zgrid)
    D = G.shape[-1]
    lq = -0.5 * zgrid ** 2 - 0.5 * LN2PI - 0.5 * np.sum((x - G) ** 2, -1) / sigma ** 2 - D * (0.5 * LN2PI + math.log(sigma))
    w = np.full(zgrid.size, zgrid[1] - zgrid[0])
    w[0] *= 0.5
    w[-1] *= 0.5
    return float(common.logsumexp(lq + np.log(w)))


def density_map(g, sigma, xs, ys, zgrid=None):
    """p(x) on a grid of points (marginal (16.81)), by quadrature over z."""
    zgrid = np.linspace(-5, 5, 801) if zgrid is None else zgrid
    G = g(zgrid)
    dz = zgrid[1] - zgrid[0]
    pz = np.exp(-0.5 * zgrid ** 2) / math.sqrt(2 * math.pi) * dz
    XX, YY = np.meshgrid(xs, ys)
    P = np.zeros_like(XX)
    for k in range(zgrid.size):
        P += pz[k] * np.exp(-((XX - G[k, 0]) ** 2 + (YY - G[k, 1]) ** 2) / (2 * sigma ** 2))
    return P / (2 * math.pi * sigma ** 2)


def mc_log_px(x, g, sigma, K, rng, R=1):
    """R independent copies of ln of the simple Monte Carlo estimate (16.83) with K prior samples."""
    out = np.zeros(R)
    D = len(x)
    for i in range(R):
        z = rng.standard_normal(K)
        G = g(z)
        l = -0.5 * np.sum((x - G) ** 2, -1) / sigma ** 2 - D * (0.5 * LN2PI + math.log(sigma))
        out[i] = common.logsumexp(l) - math.log(K)
    return out


def is_log_px(x, g, sigma, K, rng, m, s, R=1):
    """Importance-sampling estimate of ln p(x) with K draws from q = N(m, s^2)."""
    out = np.zeros(R)
    D = len(x)
    for i in range(R):
        z = m + s * rng.standard_normal(K)
        G = g(z)
        l = (-0.5 * np.sum((x - G) ** 2, -1) / sigma ** 2 - D * (0.5 * LN2PI + math.log(sigma))
             - 0.5 * z ** 2 + 0.5 * ((z - m) / s) ** 2 + math.log(s))
        out[i] = common.logsumexp(l) - math.log(K)
    return out


def mc_relvar(x, g, sigma, zgrid=ZGRID):
    """Exact relative variance var[p(x|z)]/p(x)^2 under z ~ p(z): K times the squared relative error of (16.83)."""
    G = g(zgrid)
    D = G.shape[-1]
    l = -0.5 * np.sum((x - G) ** 2, -1) / sigma ** 2 - D * (0.5 * LN2PI + math.log(sigma))
    lz = -0.5 * zgrid ** 2 - 0.5 * LN2PI + math.log(zgrid[1] - zgrid[0])
    l1 = common.logsumexp(l + lz)
    l2 = common.logsumexp(2 * l + lz)
    return math.exp(l2 - 2 * l1) - 1.0


# ==== the ELBO of the browser demo: a grid of 801 points on [-5, 5] (Riemann sum)
EGRID = np.linspace(-5.0, 5.0, 801)


def elbo_parts(x, g, sigma, m, s, zgrid=EGRID):
    """ln p(x), the ELBO (16.58) of q = N(m, s^2) and KL(q || p(z|x)) (16.59), by a Riemann sum."""
    dz = zgrid[1] - zgrid[0]
    G = g(zgrid)
    D = G.shape[-1]
    ljoint = (-0.5 * zgrid ** 2 - 0.5 * LN2PI - 0.5 * np.sum((x - G) ** 2, -1) / sigma ** 2
              - D * (0.5 * LN2PI + math.log(sigma)))
    lpx = common.logsumexp(ljoint) + math.log(dz)
    lq = -0.5 * ((zgrid - m) / s) ** 2 - 0.5 * LN2PI - math.log(s)
    q = np.exp(lq)
    elbo = float(np.sum(q * (ljoint - lq)) * dz)
    return lpx, elbo, lpx - elbo


def best_gauss_q(x, g, sigma, zgrid=EGRID):
    """Maximise the ELBO over q = N(m, s^2): coarse grid then two refinements (the demo's search)."""
    best = (-np.inf, 0.0, 0.0)
    mlo, mhi, llo, lhi = -4.0, 4.0, math.log(0.02), math.log(2.0)
    for _ in range(3):
        for i in range(41):
            m = mlo + (mhi - mlo) * i / 40
            for j in range(31):
                ls = llo + (lhi - llo) * j / 30
                e = elbo_parts(x, g, sigma, m, math.exp(ls), zgrid)[1]
                if e > best[0]:
                    best = (e, m, ls)
        dm, dl = (mhi - mlo) / 20, (lhi - llo) / 15
        mlo, mhi, llo, lhi = best[1] - dm, best[1] + dm, best[2] - dl, best[2] + dl
    return best[1], math.exp(best[2]), best[0]


# ==== a small network g(z, w) = V tanh(a z + b) + c with a one-dimensional latent space
NL_DATASETS = ("arc", "ring", "spiral", "wave")


def nlvm_data(kind, N=200, seed=3):
    """The demo's data sets, drawn with the demos' generator."""
    rng = JSRandom(seed)
    X = np.zeros((N, 2))
    for n in range(N):
        if kind == "arc":
            z = rng.norm()
            e1, e2 = rng.norm(), rng.norm()
            X[n] = [math.sin(z) + 0.1 * e1, math.cos(z) + 0.1 * e2]
        elif kind == "ring":
            t = 2 * math.pi * rng.u()
            e1, e2 = rng.norm(), rng.norm()
            X[n] = [math.cos(t) + 0.08 * e1, math.sin(t) + 0.08 * e2]
        elif kind == "spiral":
            t = rng.u()
            e1, e2 = rng.norm(), rng.norm()
            r, a = 0.25 + 1.1 * t, 3.0 * math.pi * t
            X[n] = [r * math.cos(a) + 0.05 * e1, r * math.sin(a) + 0.05 * e2]
        else:
            t = 4 * rng.u() - 2
            e1 = rng.norm()
            X[n] = [t, 0.8 * math.sin(2 * t) + 0.1 * e1]
    return X


def nlvm_init(H, seed=1, scale=1.5):
    """Initial weights from the demos' generator: a_j, b_j (j = 1..H), then V row by row; c = 0."""
    rng = JSRandom(seed)
    a, b = np.zeros(H), np.zeros(H)
    for j in range(H):
        a[j] = scale * rng.norm()
        b[j] = rng.norm()
    V = np.zeros((2, H))
    for i in range(2):
        for j in range(H):
            V[i, j] = rng.norm() / math.sqrt(H)
    return {"a": a, "b": b, "V": V, "c": np.zeros(2)}


def nlvm_g(p, z):
    h = np.tanh(np.outer(z, p["a"]) + p["b"])        # K x H
    return h @ p["V"].T + p["c"], h


def nlvm_zs(K, mode, rng=None):
    """Latent points and log weights: 'quad' = an equally spaced grid on [-4, 4] weighted by N(z|0,1);
    'mc' = K fresh draws from p(z) with weights 1/K, the estimate (16.83)."""
    if mode == "quad":
        z = np.array([-4.0 + 8.0 * (k + 0.5) / K for k in range(K)])
        lw = -0.5 * z * z
        lw = lw - common.logsumexp(lw)
    else:
        z = np.array([rng.norm() for _ in range(K)])
        lw = np.full(K, -math.log(K))
    return z, lw


def nlvm_objective(p, X, sigma, z, lw, grad=True):
    """Mean over points of ln sum_k w_k N(x_n | g(z_k), sigma^2 I) and its gradient."""
    G, h = nlvm_g(p, z)
    d2 = (X[:, None, 0] - G[None, :, 0]) ** 2 + (X[:, None, 1] - G[None, :, 1]) ** 2     # N x K
    L = lw[None, :] - d2 / (2 * sigma * sigma) - LN2PI - 2 * math.log(sigma)
    mx = L.max(1, keepdims=True)
    E = np.exp(L - mx)
    se = E.sum(1, keepdims=True)
    ll = mx[:, 0] + np.log(se[:, 0])
    f = float(ll.mean())
    if not grad:
        return f, None
    R = E / se                                                   # responsibilities N x K
    N = X.shape[0]
    Gk = np.stack([(R * (X[:, None, 0] - G[None, :, 0])).sum(0),
                   (R * (X[:, None, 1] - G[None, :, 1])).sum(0)], 1) / (sigma * sigma * N)   # K x 2: df/dg_k
    dc = Gk.sum(0)
    dV = Gk.T @ h
    dpre = (Gk @ p["V"]) * (1 - h * h)
    da = dpre.T @ z
    db = dpre.sum(0)
    return f, {"a": da, "b": db, "V": dV, "c": dc}


def nlvm_train(kind="arc", H=16, sigma=0.15, K=64, mode="quad", lr=0.02, steps=200, seed_w=1, seed_d=3,
               seed_z=11, N=200, record=None):
    """Adam ascent on the likelihood estimate; returns parameters and the objective after each step."""
    X = nlvm_data(kind, N, seed_d)
    p = nlvm_init(H, seed_w)
    rng = JSRandom(seed_z)
    mom = {k: np.zeros_like(v) for k, v in p.items()}
    vel = {k: np.zeros_like(v) for k, v in p.items()}
    b1, b2, eps = 0.9, 0.999, 1e-8
    hist = []
    zq, lwq = nlvm_zs(K, "quad")
    for t in range(1, steps + 1):
        z, lw = (zq, lwq) if mode == "quad" else nlvm_zs(K, "mc", rng)
        f, gr = nlvm_objective(p, X, sigma, z, lw)
        hist.append(f)
        for k in p:
            mom[k] = b1 * mom[k] + (1 - b1) * gr[k]
            vel[k] = b2 * vel[k] + (1 - b2) * gr[k] * gr[k]
            mh = mom[k] / (1 - b1 ** t)
            vh = vel[k] / (1 - b2 ** t)
            p[k] = p[k] + lr * mh / (np.sqrt(vh) + eps)
    return p, np.array(hist), X


def nlvm_exact_ll(p, X, sigma, K=2000):
    """The objective on a fine quadrature grid (a near-exact mean log likelihood)."""
    z, lw = nlvm_zs(K, "quad")
    return nlvm_objective(p, X, sigma, z, lw, grad=False)[0]


# =========================================================================== exports for the demos
def export_eigendigits_json(path=None, M=250):
    """Mean, eigenvalues and the first M eigenvectors (int16, base64) of the 6000 threes, plus test images."""
    X = threes()
    xbar, lam, U = pca(X)
    Um = U[:, :M].copy()
    for j in range(M):               # fix the sign: the largest-magnitude entry is positive
        if Um[np.argmax(np.abs(Um[:, j])), j] < 0:
            Um[:, j] *= -1
    scales = np.abs(Um).max(0) / 32767.0
    Q = np.round(Um / scales).astype("<i2")
    Xt, yt = common.mnist(train=False)
    pick = []
    for d, k in ((3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6), (3, 7), (7, 0), (2, 0), (8, 0), (5, 0)):
        pick.append(np.where(yt == d)[0][k])
    imgs = np.round(Xt[pick] * 255).astype(np.uint8)
    # the training example of Figure 16.5 goes first
    ex = np.round(X[FIG165_INDEX]).astype(np.uint8)
    imgs = np.vstack([ex[None, :], imgs])
    labels = [3] + [int(yt[i]) for i in pick]
    names = ["training 3 (Figure 16.5)"] + ["test %d #%d" % (labels[i + 1], i + 1) for i in range(len(pick))]
    out = {"note": "PCA of the first 6000 MNIST training threes, grey levels 0..255 (Bishop and Bishop 2024, Section 16.1.3). "
                   "U: first M eigenvectors, column j stored as int16 (row-major M x 784) times scale[j].",
           "N": int(X.shape[0]), "D": 784, "M": M,
           "mean": [round(float(v), 4) for v in xbar],
           "lam": [float("%.6g" % v) for v in lam],
           "scale": [float("%.9g" % v) for v in scales],
           "U": base64.b64encode(Q.T.tobytes()).decode("ascii"),
           "images": base64.b64encode(imgs.tobytes()).decode("ascii"),
           "labels": labels, "names": names}
    path = path or os.path.join(WEB_DATA, "ch16-eigendigits.json")
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w") as f:
        json.dump(out, f, separators=(",", ":"))
    print("wrote", path, "(%d kB)" % (os.path.getsize(path) // 1024))
    return out


def load_eigendigits(path=None):
    """Read the exported JSON back exactly as the demo does: mean, lambda, U (784 x M), images, labels."""
    path = path or os.path.join(WEB_DATA, "ch16-eigendigits.json")
    with open(path) as f:
        d = json.load(f)
    Q = np.frombuffer(base64.b64decode(d["U"]), dtype="<i2").reshape(d["M"], d["D"]).astype(float)
    U = (Q * np.array(d["scale"])[:, None]).T
    imgs = np.frombuffer(base64.b64decode(d["images"]), dtype=np.uint8).reshape(-1, d["D"]).astype(float)
    return np.array(d["mean"]), np.array(d["lam"]), U, imgs, d


FIG165_INDEX = 7   # the training three shown in Figure 16.5


def digit_error_curve(x, xbar, U, lam_rest=None):
    """Squared reconstruction error of x for M = 0..U.shape[1]: |x - xbar|^2 - sum_{i<=M} a_i^2."""
    a = U.T @ (x - xbar)
    e0 = float(np.sum((x - xbar) ** 2))
    return e0 - np.concatenate([[0.0], np.cumsum(a ** 2)])


# =========================================================================== figures
def _imshow(ax, v, title=None, cmap="gray_r", vmin=None, vmax=None, side=28):
    ax.imshow(v.reshape(side, side), cmap=cmap, vmin=vmin, vmax=vmax, interpolation="nearest")
    ax.set_xticks([])
    ax.set_yticks([])
    if title:
        ax.set_title(title, fontsize=9)


def transformed_digits(n, seed=0, size=100, digit_size=64):
    """Figure 16.1's data: an MNIST three enlarged to digit_size and placed, rotated and shifted, in a size x size field."""
    X, _ = common.mnist(train=True, digits=(3,))
    base = ndimage.zoom(X[0].reshape(28, 28), digit_size / 28.0, order=1)
    base = np.clip(base, 0, 1)
    rng = np.random.default_rng(seed)
    out = np.zeros((n, size, size))
    params = np.zeros((n, 3))
    for i in range(n):
        canvas = np.zeros((size, size))
        o = (size - digit_size) // 2
        canvas[o:o + digit_size, o:o + digit_size] = base
        ang = rng.uniform(-40, 40)
        dx, dy = rng.uniform(-14, 14, 2)
        img = ndimage.rotate(canvas, ang, reshape=False, order=1)
        img = ndimage.shift(img, (dy, dx), order=1)
        out[i] = np.clip(img, 0, 1)
        params[i] = (ang, dx, dy)
    return out, params


def fig_16_1(out_dir=None):
    """Copies of one digit, rotated and shifted at random in a 100 x 100 field: three latent variables."""
    imgs, params = transformed_digits(6, seed=4)
    fig, axes = plt.subplots(1, 6, figsize=(9.6, 1.9))
    for ax, im, pr in zip(axes, imgs, params):
        ax.imshow(im, cmap="gray_r", vmin=0, vmax=1, interpolation="nearest")
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_title(r"$%+.0f^\circ$, $(%+.0f,%+.0f)$" % tuple(pr), fontsize=8)
        for s in ax.spines.values():
            s.set_color("#999999")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-1", out_dir)


def fig_16_s1(out_dir=None):
    """PCA spectrum of 1500 transformed digits (three degrees of freedom) by the N x N route (16.28)-(16.30)."""
    imgs, _ = transformed_digits(1500, seed=1)
    X = imgs.reshape(1500, -1)
    xbar, lam, U = gram_pca(X, 1499)
    frac = np.cumsum(lam) / lam.sum()
    fig = plt.figure(figsize=(10.5, 2.9))
    left = fig.add_gridspec(1, 2, left=0.065, right=0.5, bottom=0.17, top=0.88, wspace=0.38)
    ax = fig.add_subplot(left[0, 0])
    ax.semilogy(np.arange(1, 301), lam[:300], color=common.BLUE)
    ax.set_xlabel("$i$")
    ax.set_ylabel(r"$\lambda_i$")
    ax.set_title("eigenvalues (log scale)", fontsize=9)
    ax = fig.add_subplot(left[0, 1])
    ax.plot(np.arange(1, 301), frac[:300], color=common.RED)
    for q in (0.9, 0.99):
        k = int(np.searchsorted(frac, q)) + 1
        ax.axhline(q, color=common.GREY, lw=0.6, ls=":")
        ax.plot([k], [q], "o", color="k", ms=3)
        ax.text(k + 8, q - 0.08, "%d%% at $M=%d$" % (round(100 * q), k), fontsize=8)
    ax.set_xlabel("$M$")
    ax.set_title("fraction of variance retained", fontsize=9)
    ax.set_ylim(0, 1.02)
    sub = fig.add_gridspec(2, 4, left=0.56, right=0.99, bottom=0.04, top=0.84, wspace=0.05, hspace=0.25)
    for k in range(8):
        a = fig.add_subplot(sub[k // 4, k % 4])
        v = U[:, k].reshape(100, 100)
        lim = np.abs(v).max()
        a.imshow(v, cmap="RdBu_r", vmin=-lim, vmax=lim, interpolation="nearest")
        a.set_xticks([])
        a.set_yticks([])
        a.set_title(r"$\mathbf{u}_{%d}$" % (k + 1), fontsize=8, pad=2)
    fig.text(0.775, 0.93, "first eight eigenvectors", ha="center", fontsize=9)
    return savefig(fig, CH, "fig16-s1", out_dir)


def fig_16_2(out_dir=None):
    """Projection onto the principal subspace: projected points and projection errors."""
    rng = np.random.default_rng(2)
    C = np.array([[1.0, 0.75], [0.75, 0.8]])
    X = rng.multivariate_normal([0, 0], C, 14)
    xbar, lam, U = pca(X)
    u1 = U[:, 0] if U[0, 0] > 0 else -U[:, 0]
    P = xbar + np.outer((X - xbar) @ u1, u1)
    fig, ax = plt.subplots(figsize=(4.0, 3.5))
    t = np.array([-3, 3])
    ax.plot(xbar[0] + t * u1[0], xbar[1] + t * u1[1], color=common.MAGENTA, lw=1.8)
    for x, p in zip(X, P):
        ax.plot([x[0], p[0]], [x[1], p[1]], color=common.BLUE, lw=0.9)
    ax.plot(X[:, 0], X[:, 1], "o", color=common.RED, ms=5)
    ax.plot(P[:, 0], P[:, 1], "o", color=common.GREEN, ms=4.5)
    ax.annotate("", xy=xbar + 0.9 * u1, xytext=xbar, arrowprops=dict(arrowstyle="->", lw=1.4, color="k"))
    ax.text(*(xbar + 0.9 * u1 + np.array([-0.35, 0.1])), r"$\mathbf{u}_1$", fontsize=11)
    ax.set_xlabel("$x_1$")
    ax.set_ylabel("$x_2$")
    ax.set_aspect("equal")
    ax.set_xlim(X[:, 0].min() - 0.6, X[:, 0].max() + 0.6)
    ax.set_ylim(X[:, 1].min() - 0.6, X[:, 1].max() + 0.6)
    var_proj = float(np.var((X - xbar) @ u1))
    J = float(np.mean(np.sum((X - P) ** 2, 1)))
    ax.set_title(r"projected variance %.2f $=\lambda_1$, error $J$ %.2f $=\lambda_2$" % (var_proj, J), fontsize=8.5)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-2", out_dir)


def fig_16_3(out_dir=None):
    """Mean and first four eigenvectors of 6000 MNIST threes, with eigenvalues (grey levels 0..255)."""
    X = threes()
    xbar, lam, U = pca(X)
    fig, axes = plt.subplots(1, 5, figsize=(8.6, 2.1))
    _imshow(axes[0], xbar, "mean", vmin=0, vmax=255)
    for k in range(4):
        v = U[:, k]
        if v[np.argmax(np.abs(v))] < 0:
            v = -v
        lim = np.abs(v).max()
        _imshow(axes[k + 1], v, r"$\lambda_%d = %.1f\cdot 10^5$" % (k + 1, lam[k] / 1e5), cmap="RdBu_r", vmin=-lim, vmax=lim)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-3", out_dir)


def fig_16_4(out_dir=None):
    """(a) The eigenvalue spectrum of the threes; (b) the distortion J(M), the sum of discarded eigenvalues."""
    X = threes()
    _, lam, _ = pca(X)
    J = np.array([lam[M:].sum() for M in range(785)])
    fig, axes = plt.subplots(1, 2, figsize=(8.0, 2.9))
    ax = axes[0]
    ax.plot(np.arange(1, 785), lam, color=common.BLUE)
    ax.set_xlabel("$i$")
    ax.set_ylabel(r"$\lambda_i$")
    ax.set_xlim(0, 784)
    common.panel_label(ax, "a", "upper right")
    ax = axes[1]
    ax.plot(np.arange(0, 785), J, color=common.RED)
    ax.set_xlabel("$M$")
    ax.set_ylabel("$J$")
    ax.set_xlim(0, 784)
    common.panel_label(ax, "b", "upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-4", out_dir)


def fig_16_5(out_dir=None):
    """A training three and its PCA reconstructions with M = 1, 10, 50 and 250 components."""
    X = threes()
    xbar, lam, U = pca(X)
    x = X[FIG165_INDEX]
    fig, axes = plt.subplots(1, 5, figsize=(8.6, 2.1))
    _imshow(axes[0], x, "original", vmin=0, vmax=255)
    for ax, M in zip(axes[1:], (1, 10, 50, 250)):
        r = reconstruct(x, xbar, U, M)
        err = np.sum((x - r) ** 2)
        _imshow(ax, np.clip(r, 0, 255), "$M = %d$" % M, vmin=0, vmax=255)
        ax.set_xlabel(r"$\|\mathbf{x}-\widetilde{\mathbf{x}}\|^2=%.1f\cdot10^5$" % (err / 1e5), fontsize=7.5)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-5", out_dir)


def fig_16_6(out_dir=None):
    """Old Faithful: original, standardised with principal axes, and whitened."""
    X = common.old_faithful()
    Z = (X - X.mean(0)) / X.std(0)
    xbar, lam, U = pca(Z)
    Y = (Z - xbar) @ U / np.sqrt(lam)
    fig, axes = plt.subplots(1, 3, figsize=(9.6, 3.0))
    ax = axes[0]
    ax.plot(X[:, 0], X[:, 1], "o", color=common.RED, ms=2.6, mfc="none", mew=0.8)
    ax.set_xlim(1, 6)
    ax.set_ylim(40, 100)
    ax.set_xlabel("eruption (min)")
    ax.set_ylabel("waiting (min)")
    ax = axes[1]
    ax.plot(Z[:, 0], Z[:, 1], "o", color=common.GREEN, ms=2.6, mfc="none", mew=0.8)
    for i in range(2):
        v = U[:, i] * math.sqrt(lam[i])
        ax.plot([-v[0], v[0]], [-v[1], v[1]], color=common.RED, lw=1.6)
    ax.set_aspect("equal")
    ax.set_xlim(-2.6, 2.6)
    ax.set_ylim(-2.6, 2.6)
    ax.set_title("standardised, principal axes", fontsize=9)
    ax = axes[2]
    ax.plot(Y[:, 0], Y[:, 1], "o", color=common.BLUE, ms=2.6, mfc="none", mew=0.8)
    ax.set_aspect("equal")
    ax.set_xlim(-2.6, 2.6)
    ax.set_ylim(-2.6, 2.6)
    ax.set_title("whitened (16.24)", fontsize=9)
    axes[0].set_title("original", fontsize=9)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-6", out_dir)


def fig_16_7(out_dir=None):
    """The generative view of probabilistic PCA with D = 2, M = 1: prior, p(x | z-hat) and the marginal p(x)."""
    w = np.array([1.0, 0.75])
    mu = np.array([0.0, 0.0])
    s2 = 0.12
    zh = 1.1
    fig, axes = plt.subplots(1, 3, figsize=(10.0, 3.0), gridspec_kw={"width_ratios": [1.0, 1.1, 1.1]})
    ax = axes[0]
    z = np.linspace(-3.2, 3.2, 300)
    ax.plot(z, np.exp(-z ** 2 / 2) / math.sqrt(2 * math.pi), color=common.RED)
    ax.plot([zh], [0], "o", color=common.BLUE, ms=5, clip_on=False)
    ax.axvline(zh, color=common.BLUE, lw=0.8, ls="--")
    ax.set_xlabel("$z$")
    ax.set_title(r"prior $p(z)=\mathcal{N}(0,1)$, draw $\widehat z$", fontsize=9)
    ax.set_yticks([])
    lim = (-3.0, 3.0)
    ax = axes[1]
    t = np.array([-3, 3])
    ax.plot(mu[0] + t * w[0], mu[1] + t * w[1], color=common.BLUE, lw=1.6)
    c = mu + zh * w
    for k in (1, 2):
        ax.add_patch(Circle(c, k * math.sqrt(s2), fill=False, color=common.RED, lw=1.2))
    ax.plot(*c, "o", color="k", ms=3)
    ax.plot(*mu, "o", color="k", ms=3)
    ax.text(mu[0] + 0.1, mu[1] - 0.35, r"$\boldsymbol{\mu}$")
    ax.text(c[0] - 1.2, c[1] + 0.55, r"$p(\mathbf{x}\,|\,\widehat z)$", color=common.RED)
    ax.text(2.0, 1.95, r"$\mathbf{w}$", color=common.BLUE)
    ax.set_aspect("equal")
    ax.set_xlim(*lim)
    ax.set_ylim(*lim)
    ax.set_xlabel("$x_1$")
    ax.set_ylabel("$x_2$")
    ax.set_title(r"$\mathbf{x}=\mathbf{w}\widehat z+\boldsymbol{\mu}+\boldsymbol{\epsilon}$", fontsize=9)
    ax = axes[2]
    C = np.outer(w, w) + s2 * np.eye(2)
    for k in (1, 2):
        xs, ys = common.ellipse_xy(mu, C, k)
        ax.plot(xs, ys, color=common.GREEN, lw=1.4)
    pts = np.random.default_rng(0).multivariate_normal(mu, C, 150)
    ax.plot(pts[:, 0], pts[:, 1], ".", color=common.GREY, ms=2.5)
    ax.set_aspect("equal")
    ax.set_xlim(*lim)
    ax.set_ylim(*lim)
    ax.set_xlabel("$x_1$")
    ax.set_title(r"marginal $p(\mathbf{x})=\mathcal{N}(\boldsymbol{\mu},\mathbf{w}\mathbf{w}^{\mathrm{T}}+\sigma^2\mathbf{I})$", fontsize=9)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-7", out_dir)


def fig_16_10(out_dir=None):
    """EM for PCA (16.70)-(16.71) on synthetic two-dimensional data, as alternating projections."""
    rng = np.random.default_rng(5)
    X = rng.multivariate_normal([0, 0], [[1.0, 0.82], [0.82, 1.0]], 40)
    X = (X - X.mean(0)) / X.std(0)
    xbar, lam, U = pca(X)
    W = np.array([[-0.35], [1.0]])
    Om = np.linalg.solve(W.T @ W, W.T @ X.T)
    states = [("a", None, None)]
    states.append(("b", W.copy(), Om.copy()))
    W = X.T @ Om.T @ np.linalg.inv(Om @ Om.T)
    states.append(("c", W.copy(), Om.copy()))
    Om = np.linalg.solve(W.T @ W, W.T @ X.T)
    states.append(("d", W.copy(), Om.copy()))
    W = X.T @ Om.T @ np.linalg.inv(Om @ Om.T)
    states.append(("e", W.copy(), Om.copy()))
    for _ in range(40):
        W, Om = em_pca_step(X, W)
        Om = np.linalg.solve(W.T @ W, W.T @ X.T)
    states.append(("f", W.copy(), Om.copy()))
    fig, axes = plt.subplots(2, 3, figsize=(8.4, 5.6))
    for ax, (lab, Wk, Omk) in zip(axes.ravel(), states):
        if Wk is None:
            for i in range(2):
                v = U[:, i] * math.sqrt(lam[i])
                ax.annotate("", xy=v, xytext=(0, 0), arrowprops=dict(arrowstyle="->", lw=1.5, color=common.MAGENTA))
        else:
            d = Wk[:, 0] / np.linalg.norm(Wk[:, 0])
            ax.plot([-4 * d[0], 4 * d[0]], [-4 * d[1], 4 * d[1]], color=common.RED, lw=1.5)
            P = (Wk @ Omk).T
            for x, p in zip(X, P):
                ax.plot([x[0], p[0]], [x[1], p[1]], color=common.GREY, lw=0.5)
            ax.plot(P[:, 0], P[:, 1], "o", color=common.CYAN, ms=3.2)
        ax.plot(X[:, 0], X[:, 1], "o", color=common.GREEN, ms=3.2)
        ax.set_xlim(-2.6, 2.6)
        ax.set_ylim(-2.6, 2.6)
        ax.set_aspect("equal")
        common.panel_label(ax, lab)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-10", out_dir)


def fig_16_11(out_dir=None):
    """A two-dimensional latent space mapped into three dimensions by a small tanh network with fixed weights."""
    from mpl_toolkits.mplot3d import Axes3D  # noqa: F401
    rng = np.random.default_rng(3)
    W1 = rng.normal(0, 1.0, (8, 2))
    b1 = rng.normal(0, 0.5, 8)
    W2 = rng.normal(0, 0.6, (3, 8))

    def g(Z):
        return np.tanh(Z @ W1.T + b1) @ W2.T

    u = np.linspace(-1.5, 1.5, 31)
    U1, U2 = np.meshgrid(u, u)
    P = g(np.stack([U1.ravel(), U2.ravel()], 1)).reshape(31, 31, 3)
    fig = plt.figure(figsize=(8.6, 3.4))
    ax = fig.add_subplot(1, 2, 1)
    for k in range(0, 31, 3):
        ax.plot(u, np.full_like(u, u[k]), color=common.BLUE, lw=0.6)
        ax.plot(np.full_like(u, u[k]), u, color=common.BLUE, lw=0.6)
    z0 = np.array([0.6, -0.45])
    ax.plot(*z0, "o", color=common.RED, ms=5)
    ax.text(z0[0] + 0.08, z0[1] + 0.08, r"$\mathbf{z}$", color=common.RED)
    ax.set_aspect("equal")
    ax.set_xlabel("$z_1$")
    ax.set_ylabel("$z_2$")
    ax.set_title("latent space", fontsize=9)
    ax3 = fig.add_subplot(1, 2, 2, projection="3d")
    ax3.plot_surface(P[..., 0], P[..., 1], P[..., 2], color="#9ecae1", alpha=0.55, linewidth=0, shade=True)
    for k in range(0, 31, 3):
        ax3.plot(P[k, :, 0], P[k, :, 1], P[k, :, 2], color=common.BLUE, lw=0.5)
        ax3.plot(P[:, k, 0], P[:, k, 1], P[:, k, 2], color=common.BLUE, lw=0.5)
    x0 = g(z0[None, :])[0]
    ax3.scatter([x0[0]], [x0[1]], [x0[2]], color=common.RED, s=20, depthshade=False)
    ax3.set_xlabel("$x_1$", labelpad=-8)
    ax3.set_ylabel("$x_2$", labelpad=-8)
    ax3.set_zlabel("$x_3$", labelpad=-8)
    ax3.set_xticklabels([])
    ax3.set_yticklabels([])
    ax3.set_zticklabels([])
    ax3.set_title(r"data space: $\mathbf{x}=\mathbf{g}(\mathbf{z},\mathbf{w})$", fontsize=9)
    ax3.view_init(elev=24, azim=-58)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-11", out_dir)


def fig_16_13(out_dir=None):
    """The circle model: prior p(z), the conditionals p(x|z) for three z, and the marginal p(x) (sigma = 0.3)."""
    sigma = 0.3
    xs = np.linspace(-1.8, 1.8, 181)
    fig, axes = plt.subplots(1, 5, figsize=(11.0, 2.5), gridspec_kw={"width_ratios": [1.25, 1, 1, 1, 1]})
    ax = axes[0]
    z = np.linspace(-4.5, 4.5, 300)
    ax.plot(z, np.exp(-z ** 2 / 2) / math.sqrt(2 * math.pi), color=common.RED)
    for zz in (-1.5, 0.0, 1.5):
        ax.plot([zz], [0], "o", color="k", ms=3.5, clip_on=False)
    ax.set_xlabel("$z$")
    ax.set_ylabel("$p(z)$")
    common.panel_label(ax, "a")
    XX, YY = np.meshgrid(xs, xs)
    for ax, zz in zip(axes[1:4], (-1.5, 0.0, 1.5)):
        gz = g_circle(zz)
        P = np.exp(-((XX - gz[0]) ** 2 + (YY - gz[1]) ** 2) / (2 * sigma ** 2))
        ax.imshow(P, origin="lower", extent=(xs[0], xs[-1], xs[0], xs[-1]), cmap="inferno")
        ax.set_title(r"$p(\mathbf{x}\,|\,z=%.1f)$" % zz, fontsize=9)
        ax.set_xticks([])
        ax.set_yticks([])
    P = density_map(g_circle, sigma, xs, xs)
    ax = axes[4]
    ax.imshow(P, origin="lower", extent=(xs[0], xs[-1], xs[0], xs[-1]), cmap="inferno")
    t = np.linspace(-np.pi, np.pi, 200)
    ax.plot(np.sin(t), np.cos(t), color="w", lw=0.5, ls=":")
    ax.set_title(r"marginal $p(\mathbf{x})$", fontsize=9)
    ax.set_xticks([])
    ax.set_yticks([])
    axes[1].set_xlabel("$x_1$")
    axes[1].set_ylabel("$x_2$")
    common.panel_label(axes[1], "b")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-13", out_dir)


def digit_two_variants():
    """An MNIST '2', the same image with part of the stroke removed, and the image shifted half a pixel down and right."""
    X, y = common.mnist(train=False, digits=(2,))
    a = X[0].reshape(28, 28)
    b = a.copy()
    b[12:14, 8:18] = 0.0          # a two-pixel gap across the diagonal stroke
    c = ndimage.shift(a, (0.5, 0.5), order=1, mode="constant")
    return a, b, c


def fig_16_14(out_dir=None):
    """Squared distance misleads: a damaged '2' is closer to the original than a half-pixel shift of it."""
    a, b, c = digit_two_variants()
    dab = float(np.sum((a - b) ** 2))
    dac = float(np.sum((a - c) ** 2))
    fig, axes = plt.subplots(1, 3, figsize=(6.4, 2.5))
    for ax, im, lab, t in zip(axes, (a, b, c), "abc", ("original", "part of stroke removed", "shifted by half a pixel")):
        ax.imshow(im, cmap="gray", vmin=0, vmax=1, interpolation="nearest")
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_title(t, fontsize=8.5)
        ax.set_xlabel("(%s)" % lab)
    axes[1].set_xlabel(r"(b)  $\|\mathbf{x}_a-\mathbf{x}_b\|^2=%.2f$" % dab, fontsize=8.5)
    axes[2].set_xlabel(r"(c)  $\|\mathbf{x}_a-\mathbf{x}_c\|^2=%.2f$" % dac, fontsize=8.5)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-14", out_dir)


def fig_16_15(out_dir=None):
    """Dequantisation: a distribution on {0, 1, 2} and the piecewise-constant density obtained by adding U(0, 1) noise."""
    P = np.array([0.25, 0.5, 0.25])
    fig, axes = plt.subplots(1, 2, figsize=(6.4, 2.4))
    ax = axes[0]
    ax.bar([0, 1, 2], P, width=0.12, color=common.BLUE)
    ax.set_xticks([0, 1, 2])
    ax.set_xlim(-0.5, 3.2)
    ax.set_ylim(0, 0.6)
    common.panel_label(ax, "a", "upper right")
    ax = axes[1]
    for k in range(3):
        ax.fill_between([k, k + 1], [P[k], P[k]], color=common.BLUE, alpha=0.35, step="pre")
        ax.plot([k, k + 1], [P[k], P[k]], color=common.BLUE)
    ax.set_xticks([0, 1, 2, 3])
    ax.set_xlim(-0.5, 3.2)
    ax.set_ylim(0, 0.6)
    common.panel_label(ax, "b", "upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-15", out_dir)


def fig_16_s2(out_dir=None):
    """ICA: sources, mixtures, whitened data, recovered sources, and the likelihood along rotations."""
    S, A, X, Y, V = ica_problem("sech", 500, 7)
    W, path = ica_natgrad(Y, np.eye(2), 0.1, 200, "sech")
    Z = Y @ W.T
    fig, axes = plt.subplots(1, 5, figsize=(12.4, 2.75), gridspec_kw={"width_ratios": [1, 1, 1, 1, 1.5]})
    data = [(S, "sources $\\mathbf{z}$ (16.56)", common.RED), (X, "observed $\\mathbf{x}=\\mathbf{A}\\mathbf{z}$", common.GREEN),
            (Y, "whitened (16.24)", common.BLUE), (Z, "ICA estimate", common.PURPLE)]
    for ax, (D_, t, col) in zip(axes[:4], data):
        ax.plot(D_[:, 0], D_[:, 1], ".", color=col, ms=2)
        ax.set_title(t, fontsize=9)
        ax.set_aspect("equal")
        lim = 6
        ax.set_xlim(-lim, lim)
        ax.set_ylim(-lim, lim)
    for j in range(2):
        d = A[:, j] / np.linalg.norm(A[:, j]) * 5
        axes[1].plot([-d[0], d[0]], [-d[1], d[1]], color="k", lw=0.7, ls="--")
        r = W[j] / np.linalg.norm(W[j]) * 5
        axes[2].plot([-r[0], r[0]], [-r[1], r[1]], color=common.PURPLE, lw=0.9)
    ax = axes[4]
    th = np.linspace(0, np.pi / 2, 181)
    for kind, col, lab in (("sech", common.RED, "sech sources"), ("uniform", common.BLUE, "uniform sources"),
                           ("gauss", common.GREY, "Gaussian sources")):
        _, _, _, Yk, _ = ica_problem(kind, 2000, 7)
        ll = np.array([ica_avg_loglik(rot(t), Yk, "sech") for t in th])
        ax.plot(np.degrees(th), ll - ll.mean(), color=col, label=lab)
    ax.set_xlabel(r"rotation $\theta$ of the whitened data (degrees)")
    ax.set_ylabel("log likelihood per point\n(minus its mean)", fontsize=8)
    ax.set_title("sech prior (16.56): likelihood along rotations", fontsize=9)
    ax.legend(fontsize=7, frameon=False)
    ax.set_xlim(0, 90)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-s2", out_dir)


def fig_16_s3(out_dir=None):
    """Kalman filter for z_n = a z_{n-1} + noise, x_n = z_n + noise; the gain converges to its steady state."""
    a, q, c, r = 0.95, 0.1, 1.0, 1.0
    z, x = lds_simulate(100, a, q, c, r, seed=2)
    m, P, K, ll = kalman_filter(x, a, q, c, r, 0.0, q / (1 - a * a))
    fig, axes = plt.subplots(1, 2, figsize=(10.0, 2.9), gridspec_kw={"width_ratios": [2.2, 1]})
    ax = axes[0]
    t = np.arange(100)
    ax.plot(t, x, ".", color=common.GREY, ms=3.5, label=r"observations $x_n$")
    ax.plot(t, z, color="k", lw=1.0, label=r"latent $z_n$")
    ax.plot(t, m, color=common.RED, lw=1.4, label=r"filter mean $\mathbb{E}[z_n|x_1,\ldots,x_n]$")
    ax.fill_between(t, m - 2 * np.sqrt(P), m + 2 * np.sqrt(P), color=common.RED, alpha=0.18, lw=0)
    ax.set_xlabel("$n$")
    ax.legend(fontsize=7.5, frameon=False, ncol=3, loc="upper left")
    ax.set_ylim(-4.5, 5)
    ax = axes[1]
    ax.plot(t, K, color=common.BLUE, label="gain $K_n$")
    Pinf = _riccati_fixed(a, q, c, r)
    ax.axhline(Pinf * c / (c * c * Pinf + r), color="k", lw=0.8, ls="--", label="steady state")
    ax.set_xlabel("$n$")
    ax.legend(fontsize=7.5, frameon=False)
    ax.set_title("Kalman gain", fontsize=9)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-s3", out_dir)


def _riccati_fixed(a, q, c, r):
    """Steady-state predictive variance P of the scalar filter: P = a^2 P r/(c^2 P + r) + q."""
    # c^2 P^2 + (r - a^2 r - q c^2) P - q r = 0
    A_, B_, C_ = c * c, r - a * a * r - q * c * c, -q * r
    return (-B_ + math.sqrt(B_ * B_ - 4 * A_ * C_)) / (2 * A_)


def em_ppca_threes(M=10, iters=60):
    X = threes() / 255.0
    N = X.shape[0]
    xbar = X.mean(0)
    S = (X - xbar).T @ (X - xbar) / N
    Wml, s2ml = ppca_ml(S, M)
    llml = ppca_loglik_S(S, Wml, s2ml, N)
    W, s2, hist = em_ppca(X, M, iters, seed=0, s20=1.0)
    return hist, llml, W, s2, Wml, s2ml


def fig_16_s4(out_dir=None):
    """(a) EM for probabilistic PCA on the threes approaches the closed form; (b) EM for PCA is subspace iteration."""
    hist, llml, W, s2, Wml, s2ml = em_ppca_threes(10, 300)
    X = threes() / 255.0
    N = X.shape[0]
    Xc = X - X.mean(0)
    _, lam, U = pca(X)
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 2.9))
    ax = axes[0]
    ax.plot(np.arange(1, len(hist) + 1), (llml - hist) / N, color=common.BLUE)
    ax.set_yscale("log")
    ax.set_xlabel("EM iteration")
    ax.set_ylabel(r"$[\ln p_{\rm ML}-\ln p]/N$ (nats)")
    ax.set_title("EM for probabilistic PCA, $M=10$, 6000 threes", fontsize=9)
    ax = axes[1]
    for M, col in ((1, common.RED), (5, common.BLUE), (10, common.GREEN)):
        Wk = np.random.default_rng(1).standard_normal((784, M))
        ang = []
        for _ in range(60):
            Wk, _ = em_pca_step(Xc, Wk)
            ang.append(subspace_angle(Wk, U[:, :M]))
        ang = np.array(ang)
        ax.semilogy(np.arange(1, 61), np.sin(ang), color=col, label="$M=%d$" % M)
        rate = lam[M] / lam[M - 1]
        k = np.arange(20, 61)
        ax.semilogy(k, np.sin(ang[19]) * rate ** (k - 20), color=col, lw=0.8, ls=":")
    ax.set_xlabel("EM-for-PCA iteration")
    ax.set_ylabel("sin(largest principal angle)")
    ax.set_title(r"dotted: rate $\lambda_{M+1}/\lambda_M$", fontsize=9)
    ax.legend(fontsize=7.5, frameon=False)
    fig.tight_layout()
    return savefig(fig, CH, "fig16-s4", out_dir)


def g_cyl(Z):
    """A two-dimensional latent version of the circle model: g(z1, z2) = (sin z1, cos z1, z2)."""
    return np.stack([np.sin(Z[..., 0]), np.cos(Z[..., 0]), Z[..., 1]], -1)


def mc_relvar_2d(x, sigma, n=801, lim=6.0):
    z = np.linspace(-lim, lim, n)
    Z1, Z2 = np.meshgrid(z, z, indexing="ij")
    G = g_cyl(np.stack([Z1, Z2], -1))
    l = -0.5 * np.sum((x - G) ** 2, -1) / sigma ** 2 - 3 * (0.5 * LN2PI + math.log(sigma))
    lz = -0.5 * (Z1 ** 2 + Z2 ** 2) - LN2PI + 2 * math.log(z[1] - z[0])
    l1 = common.logsumexp((l + lz).ravel())
    l2 = common.logsumexp((2 * l + lz).ravel())
    return math.exp(l2 - 2 * l1) - 1.0


def fig_16_s5(out_dir=None):
    """(a) Samples needed by (16.83) for 10% relative error against sigma; (b) ln p-hat from prior samples and from q."""
    sig = np.geomspace(0.02, 1.0, 25)
    x1 = g_circle(0.5)
    x2 = g_cyl(np.array([0.5, 0.3]))
    K1 = np.array([mc_relvar(x1, g_circle, s) / 0.01 for s in sig])
    K2 = np.array([mc_relvar_2d(x2, s, n=601) / 0.01 for s in sig])
    fig, axes = plt.subplots(1, 2, figsize=(9.4, 3.0))
    ax = axes[0]
    ax.loglog(sig, K1, "o-", color=common.RED, ms=3, label="1-D latent, circle in 2-D")
    ax.loglog(sig, K2, "s-", color=common.BLUE, ms=3, label="2-D latent, cylinder in 3-D")
    ax.loglog(sig, K1[-8] * (sig / sig[-8]) ** -1, color=common.RED, lw=0.7, ls=":")
    ax.loglog(sig, K2[-8] * (sig / sig[-8]) ** -2, color=common.BLUE, lw=0.7, ls=":")
    ax.set_xlabel(r"noise $\sigma$")
    ax.set_ylabel("$K$ for 10% relative error")
    ax.set_title(r"dotted: slopes $-1$ and $-2$ ($K\propto\sigma^{-M}$)", fontsize=9)
    ax.legend(fontsize=7.5, frameon=False)
    ax = axes[1]
    sigma = 0.05
    x = x1
    lp = log_px_quad(x, g_circle, sigma)
    rng = np.random.default_rng(0)
    a = mc_log_px(x, g_circle, sigma, 100, rng, 4000)
    m, s, _ = best_gauss_q(x, g_circle, sigma)
    b = is_log_px(x, g_circle, sigma, 100, rng, m, 1.5 * s, 4000)
    bins = np.linspace(lp - 6, lp + 1.0, 71)
    ax.hist(np.clip(a, bins[0], None), bins, color=common.RED, alpha=0.55, label="prior samples (16.83)")
    ax.hist(b, bins, color=common.BLUE, alpha=0.55, label="importance samples from $q$")
    ax.axvline(lp, color="k", lw=1)
    ax.set_xlabel(r"$\ln\widehat p(\mathbf{x})$, $K=100$, $\sigma=0.05$")
    ax.set_title(r"black: exact $\ln p(\mathbf{x})=%.3f$" % lp, fontsize=9)
    ax.legend(fontsize=7.5, frameon=False, loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-s5", out_dir)


ELBO_X = np.array([0.0, -1.0])     # the bottom of the circle: z = +pi and z = -pi both map there


def fig_16_s6(out_dir=None):
    """ELBO for a Gaussian q when the posterior is bimodal (circle model, sigma = 0.3, x = (0, -1))."""
    sigma = 0.3
    x = ELBO_X
    z = EGRID
    lpx = elbo_parts(x, g_circle, sigma, 0.0, 1.0)[0]
    ljoint = -0.5 * z ** 2 - 0.5 * LN2PI - 0.5 * np.sum((x - g_circle(z)) ** 2, -1) / sigma ** 2 - (LN2PI + 2 * math.log(sigma))
    post = np.exp(ljoint - lpx)
    m, s, e = best_gauss_q(x, g_circle, sigma)
    fig, axes = plt.subplots(1, 3, figsize=(11.0, 2.9), gridspec_kw={"width_ratios": [1.3, 1.2, 1]})
    ax = axes[0]
    ax.plot(z, np.exp(-z ** 2 / 2) / math.sqrt(2 * math.pi), color=common.GREY, lw=1, label="prior $p(z)$")
    ax.plot(z, post, color="k", label=r"posterior $p(z|\mathbf{x})$")
    ax.plot(z, np.exp(-0.5 * ((z - m) / s) ** 2) / (s * math.sqrt(2 * math.pi)), color=common.RED, label="best Gaussian $q$")
    ax.set_xlabel("$z$")
    ax.legend(fontsize=7.5, frameon=False, loc="upper left")
    ax.set_title(r"$\mathbf{x}=(0,-1)$: $z=\pm\pi$ both explain it", fontsize=9)
    ax = axes[1]
    ms = np.linspace(-4.5, 4.5, 91)
    ls = np.linspace(math.log(0.03), math.log(2.0), 61)
    E = np.array([[elbo_parts(x, g_circle, sigma, mm, math.exp(l))[1] for mm in ms] for l in ls])
    cs = ax.contourf(ms, np.exp(ls), np.maximum(E, lpx - 12), 20, cmap="viridis")
    ax.set_yscale("log")
    ax.plot([m, -m], [s, s], "x", color="w", ms=6)
    ax.set_xlabel("$m$")
    ax.set_ylabel("$s$")
    ax.set_title(r"ELBO of $q=\mathcal{N}(m,s^2)$ (clipped)", fontsize=9)
    fig.colorbar(cs, ax=ax, fraction=0.05)
    ax = axes[2]
    cases = [("prior", 0.0, 1.0), ("best\nGaussian", m, s), ("Gaussian at\none mode,\n$s=0.1$", m, 0.1)]
    vals = [elbo_parts(x, g_circle, sigma, a_, b_)[1:] for _, a_, b_ in cases]
    names = [c[0] for c in cases]
    el = np.array([v[0] for v in vals])
    kl = np.array([v[1] for v in vals])
    pos = np.arange(len(cases))
    ax.barh(pos, kl, color=common.ORANGE, label=r"KL$(q\,\|\,p(z|\mathbf{x}))=\ln p(\mathbf{x})-\mathcal{L}(q)$")
    for i in pos:
        ax.text(kl[i] + 0.15, i, "KL %.3f\nELBO %.2f" % (kl[i], el[i]), va="center", fontsize=7.5)
    ax.set_yticks(pos)
    ax.set_yticklabels(names, fontsize=7.5)
    ax.invert_yaxis()
    ax.set_xlim(0, max(kl) * 1.45)
    ax.set_xlabel("nats")
    ax.set_title(r"$\ln p(\mathbf{x})=%.3f$; KL of best $q$ = %.3f" % (lpx, kl[1]), fontsize=9)
    ax.axvline(math.log(2), color="k", lw=0.7, ls=":")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-s6", out_dir)


def _nlvm_cached(kind, steps, **kw):
    os.makedirs(CACHE_DIR, exist_ok=True)
    path = os.path.join(CACHE_DIR, "ch16_nlvm_%s_%d.npz" % (kind, steps))
    if os.path.exists(path):
        d = np.load(path)
        return {k: d[k] for k in ("a", "b", "V", "c")}, d["hist"], nlvm_data(kind)
    p, hist, X = nlvm_train(kind, steps=steps, **kw)
    np.savez(path, hist=hist, **p)
    return p, hist, X


def fig_16_s7(out_dir=None):
    """A nonlinear latent variable model (1-D latent, tanh network) fitted by maximising (16.83), against PPCA."""
    sigma = 0.15
    fig, axes = plt.subplots(2, 3, figsize=(10.0, 6.4))
    xs = np.linspace(-2.2, 2.2, 151)
    for col, kind in enumerate(("arc", "ring", "spiral")):
        p, hist, X = _nlvm_cached(kind, 3000, sigma=sigma, lr=0.02)
        ax = axes[0, col]
        P = density_map(lambda z: nlvm_g(p, z)[0], sigma, xs, xs)
        ax.imshow(P, origin="lower", extent=(xs[0], xs[-1], xs[0], xs[-1]), cmap="Greys", vmax=np.percentile(P, 99.5))
        zz = np.linspace(-3, 3, 400)
        G = nlvm_g(p, zz)[0]
        ax.plot(G[:, 0], G[:, 1], color=common.RED, lw=1.4)
        ax.plot(X[:, 0], X[:, 1], ".", color=common.BLUE, ms=2.2)
        llx = nlvm_exact_ll(p, X, sigma)
        ax.set_title("%s: network, mean ln p = %.3f" % (kind, llx), fontsize=9)
        ax.set_xticks([])
        ax.set_yticks([])
        ax = axes[1, col]
        xbar = X.mean(0)
        S = (X - xbar).T @ (X - xbar) / X.shape[0]
        W, s2 = ppca_ml(S, 1)
        C = ppca_cov(W, s2)
        llp = ppca_loglik_S(S, W, s2, X.shape[0]) / X.shape[0]
        XX, YY = np.meshgrid(xs, xs)
        D_ = np.stack([XX - xbar[0], YY - xbar[1]], -1)
        Ci = np.linalg.inv(C)
        Pp = np.exp(-0.5 * np.einsum("...i,ij,...j->...", D_, Ci, D_)) / (2 * math.pi * math.sqrt(np.linalg.det(C)))
        ax.imshow(Pp, origin="lower", extent=(xs[0], xs[-1], xs[0], xs[-1]), cmap="Greys")
        t = np.array([-3, 3])
        d = W[:, 0] / np.linalg.norm(W[:, 0])
        ax.plot(xbar[0] + t * d[0], xbar[1] + t * d[1], color=common.RED, lw=1.2)
        ax.plot(X[:, 0], X[:, 1], ".", color=common.BLUE, ms=2.2)
        ax.set_xlim(xs[0], xs[-1])
        ax.set_ylim(xs[0], xs[-1])
        ax.set_title("probabilistic PCA ($M=1$), mean ln p = %.3f" % llp, fontsize=9)
        ax.set_xticks([])
        ax.set_yticks([])
    fig.tight_layout()
    return savefig(fig, CH, "fig16-s7", out_dir)


def fig_data(out_dir=None):
    """Data file for eigen-digits-demo.html (no figure)."""
    export_eigendigits_json()


FIGURES = {"16.1": fig_16_1, "16.S1": fig_16_s1, "16.2": fig_16_2, "16.3": fig_16_3, "16.4": fig_16_4,
           "16.5": fig_16_5, "16.6": fig_16_6, "16.7": fig_16_7, "16.S2": fig_16_s2, "16.S3": fig_16_s3,
           "16.10": fig_16_10, "16.S4": fig_16_s4, "16.11": fig_16_11, "16.13": fig_16_13, "16.14": fig_16_14,
           "16.S5": fig_16_s5, "16.S6": fig_16_s6, "16.15": fig_16_15, "16.S7": fig_16_s7, "data": fig_data}


if __name__ == "__main__":
    run(FIGURES, CH)
